# ========== Basic Libraries ==========
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import time
from collections import Counter
from tabulate import tabulate

# ========== Text Processing & NLP ==========
import re  # Regular expressions for text manipulation
import nltk  # Natural Language Toolkit
nltk.download('stopwords')
nltk.download('wordnet')

from sklearn.metrics import mutual_info_score
from wordcloud import WordCloud
from nltk.corpus import stopwords  # Stopword removal library
from nltk.stem.porter import PorterStemmer  # Stemming
from gensim.models import KeyedVectors, Word2Vec  # Word embeddings
from sentence_transformers import SentenceTransformer  # Transformer-based embeddings
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer  # Text vectorization

# ========== Machine Learning ==========
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split, StratifiedKFold, GridSearchCV  # Model selection
from sklearn.preprocessing import LabelEncoder  # Encoding library for categorical labels

# ========== Machine Learning Models ==========
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier
from xgboost import XGBClassifier
from sklearn.svm import SVC  # Support Vector Classifier

# ========== Model Evaluation ==========
from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score, classification_report, confusion_matrix

# ========== Imbalanced Data Handling ==========
from imblearn.over_sampling import SMOTE
from imblearn.under_sampling import RandomUnderSampler
from imblearn.combine import SMOTEENN
os.environ["TOKENIZERS_PARALLELISM"] = "false"

[nltk_data] Downloading package stopwords to /home/opc/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package wordnet to /home/opc/nltk_data...
[nltk_data]   Package wordnet is already up-to-date!
2025-02-09 14:02:20.760234: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.
2025-02-09 14:02:20.771828: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:477] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1739109740.784968   19798 cuda_dnn.cc:8310] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1739109740.788931   19798 cuda_blas.cc:1418] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2025-02-09 14:02:20.803731: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 AVX512F AVX512_VNNI AVX512_BF16 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.

df = pd.read_excel("dataset.xlsx")
#df = pd.read_excel("C:/Users/pri96/OneDrive/Documents/AI and ML PGP/Capstone Project/NLP - 1 (Chatbot)/Data Set - industrial_safety_and_health_database_with_accidents_description.xlsx")

df.head()

# Printing shape and columns of DataFrame 'stock_news'

print("Industrial Safety DataFrame:\n")
print("     There are", df.shape[0], "rows and", df.shape[1], "columns in the dataframe\n")
print("     Columns:", df.columns.tolist())

Industrial Safety DataFrame:

     There are 425 rows and 11 columns in the dataframe

     Columns: ['Unnamed: 0', 'Data', 'Countries', 'Local', 'Industry Sector', 'Accident Level', 'Potential Accident Level', 'Genre', 'Employee or Third Party', 'Critical Risk', 'Description']

# Printing Data Types

print("\nData Types of DataFrame:\n")
print(df.dtypes)

Data Types of DataFrame:

Unnamed: 0                           int64
Data                        datetime64[ns]
Countries                           object
Local                               object
Industry Sector                     object
Accident Level                      object
Potential Accident Level            object
Genre                               object
Employee or Third Party             object
Critical Risk                       object
Description                         object
dtype: object

df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 425 entries, 0 to 424
Data columns (total 11 columns):
 #   Column                    Non-Null Count  Dtype         
---  ------                    --------------  -----         
 0   Unnamed: 0                425 non-null    int64         
 1   Data                      425 non-null    datetime64[ns]
 2   Countries                 425 non-null    object        
 3   Local                     425 non-null    object        
 4   Industry Sector           425 non-null    object        
 5   Accident Level            425 non-null    object        
 6   Potential Accident Level  425 non-null    object        
 7   Genre                     425 non-null    object        
 8   Employee or Third Party   425 non-null    object        
 9   Critical Risk             425 non-null    object        
 10  Description               425 non-null    object        
dtypes: datetime64[ns](1), int64(1), object(9)
memory usage: 36.7+ KB

df.describe(include = 'all')

#drop Unnamed column as it has no importance
df = df.drop("Unnamed: 0", axis=1)

df.head()

# Checking for duplicate values
df[df.duplicated()]

df.drop_duplicates(inplace = True)

# Checking for duplicate values again
df[df.duplicated()]

# Checking for null values
df.isnull().sum()

Data                        0
Countries                   0
Local                       0
Industry Sector             0
Accident Level              0
Potential Accident Level    0
Genre                       0
Employee or Third Party     0
Critical Risk               0
Description                 0
dtype: int64

df.rename(columns={'Data': 'Date'}, inplace=True)
df.rename(columns={'Countries': 'Country'}, inplace=True)
df.rename(columns={'Genre': 'Gender'}, inplace=True)
df.rename(columns={'Employee or Third Party': 'Employee Type'}, inplace=True)

print("\nColumn names changed to meaningful names....")
df.head()

Column names changed to meaningful names....

# Extracting year & month from 'Date' column
df['Year'] = df['Date'].dt.year
df['Month'] = df['Date'].dt.month
df['Day'] = df['Date'].dt.day_name()

df.head()

# Selecting categorical columns
categorical_columns = ["Country", "Local", "Industry Sector", "Accident Level", "Potential Accident Level", 
                       "Gender", "Employee Type", "Critical Risk", 'Year', 'Month', 'Day']

# Printing value counts for each categorical column
for col in categorical_columns:
    print(f"Value counts for {df[col].value_counts()}:\n")
    print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n")

Value counts for Country
Country_01    248
Country_02    129
Country_03     41
Name: count, dtype: int64:

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Value counts for Local
Local_03    89
Local_05    59
Local_01    56
Local_04    55
Local_06    46
Local_10    41
Local_08    27
Local_02    23
Local_07    14
Local_12     4
Local_09     2
Local_11     2
Name: count, dtype: int64:

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Value counts for Industry Sector
Mining    237
Metals    134
Others     47
Name: count, dtype: int64:

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Value counts for Accident Level
I      309
II      40
III     31
IV      30
V        8
Name: count, dtype: int64:

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Value counts for Potential Accident Level
IV     141
III    106
II      95
I       45
V       30
VI       1
Name: count, dtype: int64:

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Value counts for Gender
Male      396
Female     22
Name: count, dtype: int64:

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Value counts for Employee Type
Third Party             185
Employee                178
Third Party (Remote)     55
Name: count, dtype: int64:

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Value counts for Critical Risk
Others                                       229
Pressed                                       24
Manual Tools                                  20
Chemical substances                           17
Cut                                           14
Projection                                    13
Venomous Animals                              13
Bees                                          10
Fall                                           9
Vehicles and Mobile Equipment                  8
remains of choco                               7
Fall prevention (same level)                   7
Pressurized Systems                            7
Fall prevention                                6
Suspended Loads                                6
Liquid Metal                                   3
Pressurized Systems / Chemical Substances      3
Power lock                                     3
Blocking and isolation of energies             3
Electrical Shock                               2
Machine Protection                             2
Poll                                           1
Confined space                                 1
Electrical installation                        1
\nNot applicable                               1
Plates                                         1
Projection/Burning                             1
Traffic                                        1
Projection/Choco                               1
Burn                                           1
Projection/Manual Tools                        1
Individual protection equipment                1
Projection of fragments                        1
Name: count, dtype: int64:

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Value counts for Year
2016    283
2017    135
Name: count, dtype: int64:

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Value counts for Month
2     61
4     51
6     51
3     50
5     40
1     39
7     24
9     24
12    23
8     21
10    21
11    13
Name: count, dtype: int64:

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Value counts for Day
Thursday     76
Tuesday      69
Wednesday    62
Friday       61
Saturday     56
Monday       53
Sunday       41
Name: count, dtype: int64:

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

df['Critical Risk'] = df['Critical Risk'].str.replace(r'\nNot applicable', 'Not Applicable', regex = True)
df[df['Critical Risk']=='Not Applicable']

# Create subplots
fig, axes = plt.subplots(4, 3, figsize=(80, 100))

# List of categorical columns
categorical_columns = ['Country', 'Local', 'Industry Sector', 'Accident Level',
                        'Potential Accident Level', 'Gender', 'Employee Type', 
                        'Year', 'Month', 'Day', 'Critical Risk']

for col, ax in zip(categorical_columns, axes.flatten()):
    value_counts = df[col].value_counts()  # Get the count of unique values
    
    if len(value_counts) > 5:
        # Create countplot if the number of unique values is greater than 5
        sns.countplot(data=df, x=col, ax=ax, 
                      order=value_counts.index, hue=col, 
                      palette=sns.color_palette("Set1", len(value_counts)))
        
        ax.set_title(f'Count of {col}', fontsize=50)
        ax.set_xlabel('', fontsize=40)
        ax.set_ylabel('Count', fontsize=40)
        
        # Rotate x-tick labels and adjust font size
        ax.tick_params(axis='x', rotation=90, labelsize=40)
        ax.tick_params(axis='y', labelsize=40)
        
        # Add count labels on each bar
        for p in ax.patches:
            ax.annotate(f'{int(p.get_height())}', 
                        (p.get_x() + p.get_width() / 2., p.get_height()), 
                        ha='center', va='center',
                        fontsize=30, color='black',
                        xytext=(0, 5), textcoords='offset points')
    else:
        # Create pie chart if the number of unique values is less than or equal to 5
        value_counts.plot.pie(
            ax=ax,
            labels=value_counts.index,
            autopct='%1.1f%%',
            colors=sns.color_palette("Set1", len(value_counts)),
            textprops={'fontsize': 50, 'color': 'black'}, 
            wedgeprops={'linewidth': 2, 'edgecolor': 'black'}
        )
        
        ax.set_title(f'{col} Distribution', fontsize=50)
        ax.set_ylabel('')
        ax.set_xlabel('')
        ax.tick_params(axis='both', which='both', length=0)  # Hide ticks

# Delete any remaining empty subplots
fig.delaxes(axes[3][2])

plt.tight_layout()
plt.show()

# List of categorical columns
categorical_columns = ['Local', 'Industry Sector', 'Gender', 'Employee Type', 'Critical Risk']

# Define the number of rows and columns dynamically
num_cols = 2  # 2 columns per row
num_rows = (len(categorical_columns) + num_cols - 1) // num_cols  # Compute required rows

# Set up the grid layout dynamically
fig, axes = plt.subplots(num_rows, num_cols, figsize=(10, 15))

# Flatten the axes array to easily iterate over subplots
axes = axes.flatten()

# Dictionary for better plot titles
plot_titles = {
    'Local': 'Distribution of Local Branches by Country',
    'Industry Sector': 'Industry Sector Breakdown Across Countries',
    'Gender': 'Gender Distribution Across Countries',
    'Employee Type': 'Employee Type Comparison Across Countries',
    'Critical Risk': 'Critical Risk Levels by Country'
}

# Loop through categorical columns and create stacked bar charts
for i, col in enumerate(categorical_columns):
    value_counts = df[col].value_counts()  # Count of unique values
    count = pd.crosstab(index=df[col], columns=df['Country'])  # Crosstab for stacked bar chart

    # Create stacked bar plot
    count.plot(kind='bar', stacked=True, ax=axes[i])
    axes[i].set_title(plot_titles[col], fontsize=12)  # Improved title
    axes[i].set_xlabel('', fontsize=12)
    axes[i].set_ylabel('Count', fontsize=12)
    axes[i].set_xticklabels(axes[i].get_xticklabels(), rotation=90, fontsize=9)
    axes[i].tick_params(axis='y', labelsize=12)
    axes[i].legend(title='Country', title_fontsize=10, fontsize=9, markerscale=2)

# Remove any extra empty subplots
for j in range(i + 1, len(axes)):  
    fig.delaxes(axes[j])  # Delete extra grid spaces

# Adjust layout and display plots
plt.tight_layout()
plt.show()

fig, axes = plt.subplots(1, 4, figsize=(15, 8)) 

# List of categorical columns
categorical_columns = ['Local', 'Industry Sector', 'Employee Type', 'Critical Risk']

for i, (col, ax) in enumerate(zip(categorical_columns, axes.flatten())):
    
    # Group the data by the categorical column and 'Gender' and count occurrences
    count = df.groupby([col, 'Gender']).size().reset_index(name='Count')

    # Plot using sns.barplot
    sns.barplot(x=col, y='Count', hue='Gender', data=count, ax=ax)

    ax.set_title(f'Gender counts by {col}', fontsize=8)
    ax.set_xlabel('', fontsize=8)
    ax.set_ylabel('Count', fontsize=8)
    ax.set_xticklabels(ax.get_xticklabels(), rotation=90, fontsize=8)
    ax.tick_params(axis='y', labelsize=8)
    ax.legend(title='Gender', title_fontsize=8, fontsize=8, markerscale=2)

plt.tight_layout()
plt.show()

/tmp/ipykernel_19798/1208470090.py:17: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
  ax.set_xticklabels(ax.get_xticklabels(), rotation=90, fontsize=8)
/tmp/ipykernel_19798/1208470090.py:17: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
  ax.set_xticklabels(ax.get_xticklabels(), rotation=90, fontsize=8)
/tmp/ipykernel_19798/1208470090.py:17: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
  ax.set_xticklabels(ax.get_xticklabels(), rotation=90, fontsize=8)
/tmp/ipykernel_19798/1208470090.py:17: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
  ax.set_xticklabels(ax.get_xticklabels(), rotation=90, fontsize=8)

# Set the order for days of the week
day_order = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']

# Create a list of years
years = [2016, 2017]

# 📊 Accidents by Month (Line Plot)
fig, axes = plt.subplots(1, 2, figsize=(15, 5), sharey=True)
for i, (year, ax) in enumerate(zip(years, axes.flatten())):
    data_year = df[df['Year'] == year]
    
    # Group by 'Month' and count accidents
    monthly_accidents = data_year.groupby('Month').size()
    
    # Plotting the line plot
    sns.lineplot(x=monthly_accidents.index, y=monthly_accidents.values, ax=ax, marker='o', color='b')
    ax.set_title(f'Accidents by Month for {year}')
    
    # Set ticks and labels
    ax.set_xticks(range(len(monthly_accidents)))
    ax.set_xticklabels(monthly_accidents.index, rotation=45)
    
    ax.set_xlabel('Month')
    ax.set_ylabel('Accidents')

plt.tight_layout()
plt.show()

# 📅 Accidents by Day of the Week (Line Plot)
fig, axes = plt.subplots(1, 2, figsize=(15, 5), sharey=True)
for i, (year, ax) in enumerate(zip(years, axes.flatten())):
    data_year = df[df['Year'] == year]
    
    # Group by 'Day' and count accidents
    day_accidents = data_year.groupby('Day').size()
    
    # Reorder days
    day_accidents = day_accidents.reindex(day_order)
    
    # Plotting the line plot
    sns.lineplot(x=day_accidents.index, y=day_accidents.values, ax=ax, marker='o', color='g')
    ax.set_title(f'Accidents by Day of the Week for {year}')
    
    # Set ticks and labels
    ax.set_xticks(range(len(day_order)))
    ax.set_xticklabels(day_order, rotation=45)
    
    ax.set_xlabel('Day of the Week')
    ax.set_ylabel('Accidents')

plt.tight_layout()
plt.show()

# Encoding our dataframe
le = LabelEncoder()
df_encoded = df.apply(le.fit_transform)

# Create a larger pairplot with customized colors
sns.pairplot(df_encoded, diag_kind = 'kde', hue = 'Accident Level', height = 3, palette = 'viridis')

# Show the plot
plt.show()

# Set the figure size
plt.figure(figsize=(30, 25))

# Create the heatmap
sns.heatmap(df_encoded.corr(),  
            annot=True, 
            fmt=".2f",        # Limit decimal places
            cmap='Blues', 
            vmin=-1, vmax=1,  # Standard correlation range
            linewidths=0.5,   # Add grid lines for clarity
            annot_kws={"size": 20})     # Smaller font size for annotations

# Customize ticks and labels
plt.yticks(rotation=0, fontsize=20)
plt.xticks(rotation=90, ha='right', fontsize=20)

# Add a title
plt.title('Correlation Heatmap', fontsize=30)

# Show the plot
plt.show()

# Defining the mapping for 3-class classification
accident_mapping = {
    "I": "Low",       # Minor incidents
    "II": "Medium",   # Noticeable injuries
    "III": "Medium",  # Grouped with Level II
    "IV": "High",     # Severe injuries
    "V": "High"       # Grouped with Level IV
}

# Apply the mapping
df["Accident Category"] = df["Accident Level"].map(accident_mapping)

# Check new class distribution
class_distribution = df["Accident Category"].value_counts()
print("\nNew Class Distribution:\n", class_distribution)
df.head()

New Class Distribution:
 Accident Category
Low       309
Medium     71
High       38
Name: count, dtype: int64

# defining a function to remove special characters, lowering the case, remove extra spaces
def remove_special_characters(text):
    text = re.sub(r'[^a-zA-Z\s]', '', str(text))  # Remove special characters and numbers
    text = text.lower()  # Convert to lowercase
    text = re.sub(r'\s+', ' ', text).strip()  # Remove extra spaces
    return text

# Applying the function to remove special characters
df['Cleaned_Description'] = df['Description'].apply(remove_special_characters)

# checking a couple of instances of cleaned data
df.loc[0:6, ['Description','Cleaned_Description']]

# defining a function to remove stop words using the NLTK library
def remove_stopwords(text):
    # Split text into separate words
    words = text.split()

    # Removing English language stopwords
    new_text = ' '.join([word for word in words if word not in stopwords.words('english')])

    return new_text

# Applying the function to remove stop words using the NLTK library
df['Cleaned_Description_without_stopwords'] = df['Cleaned_Description'].apply(remove_stopwords)

# checking a couple of instances of cleaned data
df.loc[0:6, ['Cleaned_Description','Cleaned_Description_without_stopwords']]

# Loading the Porter Stemmer
ps = PorterStemmer()

# defining a function to perform stemming
def apply_porter_stemmer(text):
    # Split text into separate words
    words = text.split()

    # Applying the Porter Stemmer on every word of a message and joining the stemmed words back into a single string
    new_text = ' '.join([ps.stem(word) for word in words])

    return new_text

# Applying the function to perform stemming
df['final_cleaned_description'] = df['Cleaned_Description_without_stopwords'].apply(apply_porter_stemmer)

# checking a couple of instances of cleaned data
df.loc[0:6,['Cleaned_Description_without_stopwords','final_cleaned_description']]

# Word Cloud on the 'final_cleaned_description' column
def generate_wordcloud(df):
    text = " ".join(desc for desc in df['final_cleaned_description'])
    wordcloud = WordCloud(width=800, height=400, background_color='white').generate(text)
    plt.figure(figsize=(10, 5))
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis('off')
    plt.show()

generate_wordcloud(df)

# Frequency Plot of Words
def plot_word_frequencies(df):
    # Vectorizing the text column
    vectorizer = CountVectorizer(stop_words='english', max_features=50)  # Limit to top 50 words
    X = vectorizer.fit_transform(df['final_cleaned_description'])
    word_freq = np.asarray(X.sum(axis=0)).flatten()
    words = np.array(vectorizer.get_feature_names_out())

    # Creating a DataFrame to view word frequencies
    word_freq_df = pd.DataFrame(list(zip(words, word_freq)), columns=['word', 'frequency'])
    word_freq_df = word_freq_df.sort_values(by='frequency', ascending=False)

    # color palette for the bars
    palette = sns.color_palette("viridis", n_colors=20) 
    
    # Plotting word frequencies
    plt.figure(figsize=(12, 8))
    sns.barplot(x='frequency', y='word', data=word_freq_df.head(20), hue = 'word', palette=palette)
    plt.title('Top 20 Most Frequent Words')
    plt.xlabel('Frequency')
    plt.ylabel('Word')
    plt.show()

plot_word_frequencies(df)

# Correlation Between Frequent Words and 'Accident Level'
def correlation_with_target(df):
    # Encoding the target class ('Accident level')
    le = LabelEncoder()
    df['Accident_Level_encoded'] = le.fit_transform(df['Accident Level'])

    # Vectorizing the text column
    vectorizer = CountVectorizer(stop_words='english', max_features=50)
    X = vectorizer.fit_transform(df['final_cleaned_description'])
    y = df['Accident_Level_encoded']

    # Calculate mutual information between each word and the target class
    mi_scores = []
    for i in range(X.shape[1]):
        mi_scores.append(mutual_info_score(X[:, i].toarray().flatten(), y))

    # Creating a DataFrame to view mutual information scores
    mi_df = pd.DataFrame(list(zip(vectorizer.get_feature_names_out(), mi_scores)), columns=['word', 'MI Score'])
    mi_df = mi_df.sort_values(by='MI Score', ascending=False)

    # color palette for the bars
    palette = sns.color_palette("coolwarm", n_colors=20)
    
    # Plotting mutual information scores
    plt.figure(figsize=(12, 8))
    sns.barplot(x='MI Score', y='word', data=mi_df.head(20), hue = 'word', palette=palette)
    plt.title('Top 20 Words with Highest Mutual Information with Accident Level')
    plt.xlabel('Mutual Information Score')
    plt.ylabel('Word')
    plt.show()

correlation_with_target(df)

df = df.drop("Date", axis = 1)
df = df.drop("Accident Level", axis = 1)
df = df.drop("Potential Accident Level", axis = 1)

df.head()

# Identify categorical columns
categorical_cols = ["Country", "Local", "Industry Sector",
                    "Gender", "Employee Type", "Critical Risk", "Accident Category"]

# Apply Label Encoding to all categorical columns
label_encoders = {}

for col in categorical_cols:
    le = LabelEncoder()
    df[col] = le.fit_transform(df[col])
    label_encoders[col] = le  # Store the encoder for potential inverse transformations

# Display updated DataFrame
df.head()

X = df.drop(columns = ['Accident Category'])  # Predictors (all columns except 'Accident Category')
y = df['Accident Category']  # Target variable, 'Accident Category'

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, stratify = y, random_state = 42)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.25, stratify = y_train, random_state = 42)

X_train = np.array(X_train)
X_test = np.array(X_test)
X_val = np.array(X_val)
y_train = np.array(y_train)
y_test = np.array(y_test)
y_val = np.array(y_val)

print(f"\nNumber of training samples: {len(X_train)}")
print(f"Number of testing samples: {len(X_test)}")
print(f"Number of validation samples: {len(X_val)}")

# Printing the shapes of the train and test sets
print("\nShape of X_train:", X_train.shape)
print("Shape of X_test:", X_test.shape)
print("Shape of X_val:", X_val.shape)
print("\nShape of y_train:", y_train.shape[0])
print("Shape of y_test:", y_test.shape[0])
print("Shape of y_val:", y_val.shape[0])

Number of training samples: 250
Number of testing samples: 84
Number of validation samples: 84

Shape of X_train: (250, 14)
Shape of X_test: (84, 14)
Shape of X_val: (84, 14)

Shape of y_train: 250
Shape of y_test: 84
Shape of y_val: 84

# Creating a list of all words in our data
list_of_words = [item.split(" ") for item in df['final_cleaned_description'].values]

flat_list_of_words = [word for sublist in list_of_words for word in sublist]

print("\n Number of words in our data - ", len(flat_list_of_words))

 Number of words in our data -  13712

# Creating an instance of Word2Vec

word2vec_model = Word2Vec(list_of_words,
                          vector_size = 200,
                          min_count = 1,
                          window = 5,
                          workers = 6)
word2vec_model

<gensim.models.word2vec.Word2Vec at 0x7efd40016690>

# Checking the size of the vocabulary
print("Length of the vocabulary is", len(list(word2vec_model.wv.key_to_index)))

Length of the vocabulary is 2284

# Checking the word embedding of a random word
random_word = 'accident'
word2vec_model.wv[random_word]

array([-3.37621226e-04,  3.77410068e-03,  1.79451681e-03,  3.77663132e-03,
       -1.91464764e-03, -1.96212973e-03, -3.68073769e-03,  6.96802698e-03,
       -1.30749086e-03,  2.99211522e-03, -6.79945981e-04, -3.08721000e-03,
        2.10195594e-03,  9.42128245e-04, -4.98327985e-03, -6.35044184e-03,
        2.49802484e-03, -4.40180441e-03,  1.51990971e-03,  1.02162291e-03,
        4.44159051e-03, -6.70778845e-03,  1.06399483e-03, -9.72829468e-04,
        3.96361342e-03, -9.49141162e-04,  3.99223249e-03, -6.42957748e-04,
       -4.55213711e-03, -7.09228625e-04, -2.50244001e-03,  1.06981699e-03,
       -1.09699764e-03, -3.48456600e-03,  1.84537517e-03,  6.29364373e-03,
        5.22836344e-03,  4.63317055e-03, -5.12305507e-03, -1.83835113e-03,
        2.18195701e-03,  4.08131833e-04,  1.81924191e-03, -2.63074669e-03,
        7.51104718e-03,  2.92979437e-03, -1.37394271e-03,  3.05605214e-03,
       -1.45575858e-03,  2.82122963e-03,  9.76301206e-04,  1.25642528e-03,
       -3.96670774e-03, -4.06727137e-04, -7.62477401e-04,  2.46713357e-03,
       -3.62140802e-03, -2.66518048e-03, -6.26920210e-03,  9.54528950e-05,
       -4.39789612e-03, -9.68957029e-04, -4.03250940e-03, -1.94379920e-03,
       -1.35483162e-03,  1.11049296e-04, -3.88162443e-03,  8.07375833e-03,
       -1.71863101e-03,  4.11973195e-03, -3.24488647e-04, -2.83252541e-03,
        1.21856597e-03,  3.53821227e-03, -8.87529517e-04,  3.56177450e-03,
        2.23255157e-03, -4.69599618e-03, -6.03192346e-03, -1.71901454e-04,
       -3.73012852e-03, -4.62651625e-03,  1.08555565e-03,  5.62686427e-03,
        3.74637515e-04,  4.07951977e-03, -1.94159930e-03,  4.31664055e-03,
        1.45220046e-03,  7.04006699e-04,  1.94029592e-03,  1.12681868e-04,
       -3.23016034e-03,  3.48559814e-03,  7.35582737e-03, -1.53158617e-04,
        3.88680631e-03, -7.93682295e-04, -1.28883659e-03, -3.81403137e-03,
       -6.17611827e-03,  2.13742163e-03, -3.49545456e-03, -2.59866868e-03,
       -3.86867020e-03, -5.45459241e-03,  1.96950440e-03,  4.28698864e-03,
       -5.95217990e-03, -3.98606388e-03, -1.19504193e-03, -5.26390225e-03,
       -2.44331174e-03, -4.25592391e-03,  3.20336083e-03,  2.93504307e-03,
       -1.80748582e-03, -6.60018763e-03, -4.58070729e-03, -4.71243868e-03,
        5.33711724e-03, -1.73371146e-03,  2.77490797e-03, -1.68202107e-03,
       -4.66003502e-03, -2.99090473e-03, -4.70825424e-03, -1.81823841e-03,
        2.16340460e-03,  5.51813864e-04,  1.97475101e-03, -1.70459738e-03,
       -4.23609745e-03, -1.61565212e-03, -1.59941812e-03,  6.61158515e-03,
        1.48112676e-03, -4.89364052e-03, -2.83321249e-03, -6.58255070e-03,
        3.12008779e-03, -5.96832437e-03, -1.42878818e-03, -5.28929173e-04,
       -4.66451282e-03, -6.06763596e-03, -5.31379506e-03,  2.27213418e-03,
       -1.67837588e-03,  3.79696465e-03, -4.43842588e-03,  1.93250482e-03,
        4.29942366e-03,  6.14917604e-03, -7.81808048e-04, -4.15079092e-04,
        5.59904566e-03, -2.03870214e-03, -3.96646233e-03,  4.97209560e-03,
        6.41897321e-04, -2.59614922e-03, -9.93015943e-04, -1.80885254e-03,
       -4.70476272e-03,  4.91394475e-03,  3.03823967e-03, -1.34907826e-03,
        3.36757023e-03, -1.99919869e-03, -5.36392210e-03,  6.62274472e-03,
       -3.77108721e-04, -4.98923939e-03,  3.80086526e-03,  6.24962477e-03,
       -1.43956312e-03,  2.02047871e-03,  4.89682934e-05,  4.59815987e-04,
        3.98806483e-03,  4.52615612e-04,  3.90779600e-03, -3.02215689e-03,
        2.92674033e-03, -1.87033741e-03, -4.14725998e-03,  4.52509150e-03,
        5.16260136e-03,  2.13559531e-03, -2.58347573e-04,  1.62634382e-03,
       -3.62123130e-03, -6.62840204e-03,  2.03037611e-03, -2.44232873e-03,
       -2.90385075e-03,  6.41431601e-04,  3.60921887e-03, -5.65241615e-04],
      dtype=float32)

# checking words similar to a random word. 'employee' in below case

similar_words = word2vec_model.wv.most_similar(random_word, topn = 5)
print(f"\nWords similar to '{random_word}':", similar_words)

Words similar to 'accident': [('produc', 0.6144047975540161), ('identifi', 0.611992359161377), ('proceed', 0.6071212291717529), ('fifth', 0.6070601344108582), ('center', 0.6066087484359741)]

def get_avg_word2vec(text_in_list_of_words, model, vector_size):
    """
    Returns the average Word2Vec vector for a text
    text_in_list_of_words: This is the individual text which needs to be averaged to a single vector
    model: This is the initialized and trained Word2Vec model
    vector_size: The size of final averaged vector
    """
    valid_words = [model.wv[word] for word in text_in_list_of_words if word in model.wv]
    if len(valid_words) > 0:
        return np.mean(valid_words, axis = 0)
    else:
        return np.zeros(vector_size)

# Creating embeddings for the full dataset and storing in a new column, 'Word2Vec_Embedding'
text_to_words_list = df['final_cleaned_description'].apply(lambda text: text.split())

df['Word2Vec_Embedding'] = text_to_words_list.apply(lambda words_list: 
                                                            get_avg_word2vec(words_list, word2vec_model, 100))
df.head()

glove_file = "glove.6B.100d.txt.word2vec"

#Loading the GloVe Model
glove_model = KeyedVectors.load_word2vec_format(glove_file, binary = False)

glove_model

<gensim.models.keyedvectors.KeyedVectors at 0x7efd400ac530>

# Checking the size of the vocabulary
print("Length of the vocabulary is", len(glove_model.index_to_key))

Length of the vocabulary is 400000

# Checking the word embedding of a random word
random_word = "accident"
glove_model[random_word]

array([-6.3287e-03, -3.7913e-01,  4.0992e-01, -3.8438e-03, -8.1139e-01,
       -6.7840e-01,  2.5995e-01,  1.0903e+00,  6.0039e-01,  6.8617e-02,
        6.0529e-01,  9.9349e-01,  6.2516e-01, -6.4927e-02, -1.3299e-01,
       -1.1893e-01,  7.2227e-02, -2.9411e-01, -5.2764e-01,  7.9741e-01,
        4.9420e-01, -8.8902e-02,  2.3917e-01, -1.3490e-04,  1.0083e-01,
        3.0154e-01, -8.5693e-01,  6.0000e-01,  3.9684e-01,  2.3637e-01,
       -5.3385e-01,  2.2272e-02, -1.1326e-01,  7.8765e-02, -9.8925e-01,
       -4.5780e-01, -1.6784e-01,  3.2173e-02,  3.1255e-01,  5.6557e-01,
       -3.1221e-01,  3.2615e-01,  2.5084e-01, -6.2934e-01, -1.8363e-03,
        6.9551e-01,  5.7464e-01,  4.4739e-02, -2.4990e-01, -7.9119e-01,
        5.2257e-01, -2.1110e-01, -1.6103e-01,  9.8529e-01,  1.2813e-01,
       -1.5351e+00, -5.6295e-01,  4.1564e-01,  1.8002e+00,  1.0223e+00,
       -1.4897e-01,  9.9298e-01,  5.4087e-01,  2.7504e-01,  4.1355e-01,
        2.8260e-01, -3.0770e-01, -6.3867e-01, -3.3489e-01,  3.9878e-01,
       -1.1430e+00, -1.6836e-01,  2.9948e-01,  6.6810e-01,  7.4680e-01,
        3.4709e-01,  1.3212e+00, -1.4027e-01, -8.9800e-01, -1.9512e-01,
        1.2478e-01,  5.9976e-01,  3.7032e-01, -1.5821e-01, -8.5888e-01,
       -3.3328e-01, -3.4937e-01,  1.2206e-01, -1.0733e+00, -1.7204e-01,
        4.9451e-01, -3.0870e-01, -1.8550e-01,  7.1409e-01,  1.9886e-01,
        1.1276e+00, -1.0096e-01, -1.0000e-01,  2.1349e-01, -1.2453e+00],
      dtype=float32)

def get_avg_glove(text_in_list_of_words, model, vector_size):
    """
    Returns the average Word2Vec vector for a text
    text_in_list_of_words: This is the individual text which needs to be averaged to a single vector
    model: This is the initialized and trained GloVe model
    vector_size: The size of final averaged vector
    """
    valid_words = [model[word] for word in text_in_list_of_words if word in model]
    if len(valid_words) > 0:
        return np.mean(valid_words, axis = 0)
    else:
        return np.zeros(vector_size)

# Creating embeddings for the full dataset and storing in a new column, 'GloVe_Embedding'
text_to_words_list = df['final_cleaned_description'].apply(lambda text: text.split())

df['GloVe_Embedding'] = text_to_words_list.apply(lambda words_list: 
                                                         get_avg_glove(words_list, glove_model, 100))
df.head()

# Defining the model
sentence_transformer_model = SentenceTransformer('all-MiniLM-L6-v2')

sentence_transformer_model

SentenceTransformer(
  (0): Transformer({'max_seq_length': 256, 'do_lower_case': False}) with Transformer model: BertModel 
  (1): Pooling({'word_embedding_dimension': 384, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})
  (2): Normalize()
)

# Getting embedding for a single Description Text
random_news = df['final_cleaned_description'].iloc[0]
embedding = sentence_transformer_model.encode(random_news)
print("\n Cleaned News Article Example - ", random_news)
print("\nSentence Transformer Embedding for first news article:\n", embedding)

 Cleaned News Article Example -  remov drill rod jumbo mainten supervisor proce loosen support intermedi central facilit remov see mechan support one end drill equip pull hand bar acceler remov moment bar slide point support tighten finger mechan drill bar beam jumbo

Sentence Transformer Embedding for first news article:
 [-5.83910570e-02 -1.85151901e-02 -2.64099729e-03 -1.52495233e-02
 -1.09215379e-01  2.93994471e-02  2.39257030e-02  4.32314537e-02
 -6.29347116e-02  2.88824160e-02  4.29037139e-02 -1.20463665e-04
 -3.51383984e-02 -1.16187520e-02 -6.05066717e-02  5.74241504e-02
 -2.93129473e-04  3.13549526e-02  1.09500624e-02 -3.69974598e-02
  5.19531488e-04 -1.84541848e-02 -1.34072667e-02 -2.68476959e-02
 -1.06333524e-01  5.23765683e-02 -6.04821444e-02  2.09673010e-02
  3.76024097e-02 -9.81916413e-02 -2.43850891e-02  1.72222126e-02
  9.55908559e-03 -3.16833034e-02 -4.55957912e-02  6.11252822e-02
 -1.95393916e-02 -1.23174954e-02 -3.82162072e-02 -6.29551942e-03
 -1.45713715e-02  2.21917704e-02  3.67429741e-02 -6.99436143e-02
  7.73041993e-02  1.06778681e-01 -1.65814478e-02 -2.76878178e-02
  1.02433555e-01  2.31582928e-03  4.28303285e-03 -2.19484717e-02
  8.00658856e-03 -1.48252035e-02  6.01477288e-02 -3.84836830e-02
  5.12675010e-02  3.89207788e-02  1.32774577e-01  6.16852567e-03
  4.62846123e-02  7.17263147e-02 -4.35719565e-02  9.64082778e-03
  9.94112808e-03 -4.67867702e-02 -2.66925972e-02 -2.86831316e-02
  6.04374846e-03  1.95276388e-03  5.79159409e-02 -1.80625375e-02
 -1.09912738e-01 -3.03124660e-03  4.82812375e-02 -7.86706340e-03
 -3.73575673e-03 -1.74115524e-02  6.10726215e-02  2.01896001e-02
 -6.29363433e-02  2.17748471e-02 -2.39128713e-02  3.69077139e-02
  6.15670048e-02 -5.24854586e-02 -2.58085560e-02  1.07724303e-02
  9.33566168e-02  3.62559892e-02  3.69745530e-02 -4.54940721e-02
 -2.80765966e-02  2.92164627e-02  5.35804108e-02  1.60183143e-02
 -7.37158582e-02  5.72818208e-05 -9.84754637e-02  3.80433574e-02
  6.05139025e-02 -1.29120816e-02  2.58483626e-02 -1.68156382e-02
 -1.11925349e-01  4.24383916e-02 -3.47675160e-02  4.78190109e-02
 -4.30331789e-02  3.16276327e-02  2.08172249e-03 -4.64619920e-02
 -1.07553760e-05 -9.65625048e-02 -2.72715837e-02 -1.63166616e-02
 -7.20350742e-02  5.58371916e-02 -5.67682348e-02 -6.45558536e-02
  2.15483569e-02 -4.96683549e-03 -1.31739173e-02 -1.20625630e-01
  5.19372746e-02 -1.42644346e-03  1.62340589e-02  1.95011074e-32
  4.08573495e-03 -9.21876132e-02 -1.10725649e-01  6.66885599e-02
  3.79700847e-02 -3.65715325e-02 -1.84197805e-03  3.63599882e-02
  1.66347306e-02 -1.25615560e-02  6.58446923e-02  6.78747296e-02
 -8.54164436e-02 -3.81455310e-02 -4.82569309e-03 -9.13610235e-02
  3.75232771e-02  6.24974666e-04 -5.25042862e-02  8.98917019e-03
 -2.36967131e-02  2.37992592e-02 -4.46925685e-02 -7.29526160e-03
  6.46198913e-02  3.68542820e-02  1.10231049e-01  5.44541851e-02
 -1.18744247e-01  3.51226516e-02 -3.42020877e-02  5.00015393e-02
  2.98795439e-02 -2.41703503e-02  2.34902464e-02  2.54273620e-02
  5.69192134e-02 -7.11193085e-02 -1.04383349e-01 -7.83880875e-02
  4.70761620e-02 -1.70444101e-02 -1.26159862e-01 -1.59544330e-02
  3.58857103e-02 -1.14725437e-02  7.56672919e-02  1.29260344e-03
  1.43594043e-02 -1.37028927e-02 -4.22023721e-02  5.40786907e-02
  2.82780249e-02 -1.08418558e-02 -3.35904807e-02  2.65133008e-03
 -1.12826386e-02  9.25617740e-02  1.29699046e-02  2.56848186e-02
 -4.88745794e-02  6.22059442e-02  3.31431925e-02  1.02616757e-01
  4.68709953e-02 -1.06090061e-01 -1.68832690e-02 -4.15779091e-02
  8.06282237e-02  7.45307235e-03 -4.99228649e-02 -2.43548974e-02
 -9.28648189e-03 -3.21599543e-02 -1.20629482e-01  3.05741224e-02
 -2.62945332e-02  5.36918975e-02 -5.62039763e-02 -3.20742279e-02
 -4.59024422e-02  2.99480762e-02  2.44402345e-02  2.39482094e-02
  5.15885763e-02 -7.94841498e-02  3.84179056e-02 -1.93864806e-03
  6.94416557e-03 -1.70165077e-02  1.53702199e-02 -5.85890561e-02
  4.41317819e-03  6.63740933e-02  1.72843237e-03 -1.77221540e-32
 -6.99837580e-02  4.02151328e-03  4.89845127e-02  2.03272980e-02
  1.59983169e-02  3.20430705e-03  2.93471199e-02 -2.74604224e-02
 -1.76774058e-02 -1.37245767e-02 -3.38780582e-02 -5.44712655e-02
  2.30156779e-02 -1.47452876e-02  4.64714840e-02  1.00586645e-01
 -9.20863599e-02  3.39630619e-02 -1.03044003e-01  4.46662791e-02
  5.08427396e-02  7.77949765e-02  5.26272655e-02  5.94687425e-02
  4.10053134e-02  9.49665066e-03 -6.66515753e-02 -1.81449018e-02
 -2.91736815e-02  4.40373272e-02 -3.55774611e-02 -9.40166116e-02
 -1.32857403e-02  6.42624944e-02 -5.07948250e-02 -1.39048658e-04
  2.28683315e-02  2.32309457e-02  9.69106425e-03 -1.47188203e-02
  8.84755999e-02  1.49783164e-01 -4.67827031e-03 -6.15788903e-03
 -7.72532001e-02 -4.40657921e-02  6.29334226e-02 -3.19874249e-02
 -1.84668563e-02 -3.67270643e-03 -6.58613443e-02 -3.92466709e-02
 -1.80639047e-02 -6.88584670e-02  4.60533276e-02  2.55073397e-03
  4.15919386e-02 -8.90576988e-02 -5.46911545e-02  2.88098119e-02
  4.97676879e-02 -2.66624149e-02  3.03503536e-02  4.12735790e-02
  7.62828961e-02 -6.54989714e-03  6.35135248e-02 -2.12884527e-02
 -9.86543372e-02 -5.73263876e-03  1.92341767e-03 -8.17120820e-03
  1.09346569e-01 -4.93310057e-02  7.91552290e-02 -5.26485629e-02
 -4.38191667e-02 -6.78109899e-02 -2.04692613e-02 -2.72623599e-02
  4.39879671e-02 -9.09667984e-02  1.30756693e-02  6.97067901e-02
 -7.98465908e-02  5.99958971e-02 -4.48753163e-02  7.07621127e-02
  2.97195483e-02  7.94801302e-03 -3.00834309e-02  2.47003697e-02
  8.83330181e-02  3.64643261e-02  1.34203890e-02 -4.82048357e-08
 -7.24332258e-02 -9.67439543e-03 -2.36218306e-03 -1.52342804e-02
  7.35293552e-02  5.57801779e-03 -8.64913985e-02  6.13728277e-02
 -4.21949849e-02  2.80490005e-03  8.20408612e-02 -5.59826940e-02
 -2.58966219e-02  3.19656692e-02  4.32729498e-02  1.60948765e-02
 -5.77330478e-02  8.38913172e-02 -3.10840942e-02 -1.21312410e-01
  3.13505307e-02 -8.32298398e-02  1.42268434e-01  5.84231578e-02
 -6.80748299e-02 -1.56990252e-02 -9.28214192e-02  1.36467228e-02
  7.55442167e-03  5.03864326e-02 -2.54175887e-02  2.59978697e-03
  3.04667782e-02 -6.89875335e-02  2.07286309e-02  3.84675898e-02
 -8.17425363e-03  4.89655733e-02  1.11290980e-02  3.37486304e-02
 -8.35158303e-03  1.67263355e-02 -9.64002218e-03  7.83861130e-02
 -9.67168137e-02 -6.82592718e-03 -2.54861582e-02  2.70241704e-02
 -7.65214935e-02 -7.95234218e-02  2.80866679e-02 -4.69980910e-02
  1.00820893e-02 -4.23919149e-02  2.53251959e-02  5.63129075e-02
 -2.08844338e-02  3.63378003e-02 -2.73159165e-02  2.11154334e-02
 -6.13151537e-03  6.14760704e-02 -3.94358374e-02 -7.73836151e-02]

# Generating averaged embeddings for each Description Text
start_time = time.time()
df['SentenceTransformer_Embedding'] = df['final_cleaned_description'].apply(lambda text: 
                                                                               sentence_transformer_model.encode(text))
end_time = time.time()

total_time = end_time - start_time

if(total_time < 60):
    print(f"\n Total time taken for generating averaged embeddings using Sentence Transformer - {total_time:.2f} seconds")
else:
    print(f"\n Total time taken for generating averaged embeddings using Sentence Transformer - ",
          int(total_time / 60), "minute(s)", int(total_time % 60), "second(s)")

df.head()

 Total time taken for generating averaged embeddings using Sentence Transformer - 2.80 seconds

# Initialize TF-IDF Vectorizer
tfidf_vectorizer_model = TfidfVectorizer()

tfidf_vectorizer_model

TfidfVectorizer()

TfidfVectorizer()

# Fit and transform the text data
tfidf_embeddings = tfidf_vectorizer_model.fit_transform(df['final_cleaned_description'])

# Getting feature names (words)
feature_names = tfidf_vectorizer_model.get_feature_names_out()

# Convert TF-IDF matrix to DataFrame for readability
tfidf_df = pd.DataFrame(tfidf_embeddings.toarray(), columns = feature_names)

# Generating averaged embeddings for each Description Text

df['TFIDF_Embedding'] = list(tfidf_embeddings.toarray())

df.head()

# Initialize CountVectorizer
bow_model = CountVectorizer() 

bow_model

CountVectorizer()

CountVectorizer()

# Fit & transform the text data
bow_model_matrix = bow_model.fit_transform(df['final_cleaned_description'])

# Get feature names (words)
bow_features = bow_model.get_feature_names_out()

# Convert to a DataFrame (BoW representation)
bow_df = pd.DataFrame(bow_model_matrix.toarray(), columns = bow_features)

#  Convert to a dense matrix and average across columns (words)
#bow_embeddings = bow_df.mean(axis = 1)
bow_embeddings = bow_model_matrix.toarray().mean(axis = 1)
# Store the BoW embeddings in the main DataFrame
df['BoW_Embedding'] = bow_embeddings

df.head()

X_word2vec = np.stack(df['Word2Vec_Embedding'].values)
X_glove = np.stack(df['GloVe_Embedding'].values)
X_sentence_transformer = np.stack(df['SentenceTransformer_Embedding'].values)
X_tfidf = np.stack(df['TFIDF_Embedding'].values)
X_bow = np.array(df['BoW_Embedding'].values).reshape(-1, 1) # reshaping 1D embeddings into 2D to match the dimensions
y = df['Accident Category'].values

# Spliting train, test and validation sets
X_train_idx, X_test_idx, y_train, y_test = train_test_split(
    range(len(y)), y, test_size = 0.2, stratify = y, random_state = 42)

X_train_idx, X_val_idx, y_train, y_val = train_test_split(
    X_train_idx, y_train, test_size = 0.25, stratify = y_train, random_state = 42)

# Applying the same split indices to all embeddings
X_train_word2vec = X_word2vec[X_train_idx]
X_val_word2vec = X_word2vec[X_val_idx]
X_test_word2vec = X_word2vec[X_test_idx]

X_train_glove = X_glove[X_train_idx]
X_val_glove = X_glove[X_val_idx]
X_test_glove = X_glove[X_test_idx]

X_train_sentence_transformer = X_sentence_transformer[X_train_idx]
X_val_sentence_transformer = X_sentence_transformer[X_val_idx]
X_test_sentence_transformer = X_sentence_transformer[X_test_idx]

X_train_tfidf = X_tfidf[X_train_idx]
X_val_tfidf = X_tfidf[X_val_idx]
X_test_tfidf = X_tfidf[X_test_idx]

X_train_bow = X_bow[X_train_idx]
X_val_bow = X_bow[X_val_idx]
X_test_bow = X_bow[X_test_idx]


# Printing shapes to confirm - Creating a dictionary to hold the information
data = {
    "Embedding Type": ["Word2Vec", "GloVe", "Sentence Transformer", "TF-IDF", "BoW"],
    #"Embedding Type": ["Word2Vec", "GloVe", "Sentence Transformer", "TF-IDF"],
    "Training Samples": [
        len(X_train_word2vec),
        len(X_train_glove),
        len(X_train_sentence_transformer),
        len(X_train_tfidf),
        len(X_train_bow)
    ],
    "Validation Samples": [
        len(X_val_word2vec),
        len(X_val_glove),
        len(X_val_sentence_transformer),
        len(X_val_tfidf),
        len(X_val_bow)
    ],
    "Testing Samples": [
        len(X_test_word2vec),
        len(X_test_glove),
        len(X_test_sentence_transformer),
        len(X_test_tfidf),
        len(X_test_bow)
    ],
    "Training Shape": [
        str(X_train_word2vec.shape),
        str(X_train_glove.shape),
        str(X_train_sentence_transformer.shape),
        str(X_train_tfidf.shape),
        str(X_train_bow.shape)
    ],
    "Validation Shape": [
        str(X_val_word2vec.shape),
        str(X_val_glove.shape),
        str(X_val_sentence_transformer.shape),
        str(X_val_tfidf.shape),
        str(X_val_bow.shape)
    ],
    "Testing Shape": [
        str(X_test_word2vec.shape),
        str(X_test_glove.shape),
        str(X_test_sentence_transformer.shape),
        str(X_test_tfidf.shape),
        str(X_test_bow.shape)
    ],
}

evaluation_df = pd.DataFrame(data)

print("\nTrain, Validation, and Test Set Summary:")
evaluation_df.head()

Train, Validation, and Test Set Summary:

le = LabelEncoder()
y_train = le.fit_transform(y_train)
y_test = le.fit_transform(y_test)
y_val = le.fit_transform(y_val)

def apply_resampling(X_train, y_train, strategy = 'both', embedding_type = 'unknown'):
    """
    Applies SMOTE and undersampling techniques to balance the dataset
    
    Parameters:
    X_train: Training set of intependent variables
    y_train: Training set of target variable
    strategy: String type. 'both' for SMOTE + Undersampling, 'smote' for only SMOTE
    embedding_type: String type. Representing the embedding technique used

    Returns:
    X_resampled, y_resampled: Balanced sets for independent and target variables
    distribution_df: DataFrame summarizing class distributions before and after resampling
    """
    original_distribution = Counter(y_train)
    # Convert any datetime columns to numeric timestamps
    if isinstance(X_train, pd.DataFrame):
        for col in X_train.select_dtypes(include=['datetime64[ns]']):
            print("Timestamp - ", col)
            #X_train[col] = X_train[col].astype(int) / 10**9  # Convert to seconds
    smote = SMOTE(sampling_strategy = 'auto', random_state = 42, k_neighbors = 1)
    
    if strategy == 'both':
        undersample = RandomUnderSampler(sampling_strategy = 'auto', random_state = 42)
        X_smote, y_smote = smote.fit_resample(X_train, y_train)
        X_resampled, y_resampled = undersample.fit_resample(X_smote, y_smote)
    elif strategy == 'smote':
        X_resampled, y_resampled = smote.fit_resample(X_train, y_train)
    else:
        raise ValueError("Invalid strategy. Use 'both' or 'smote'.")
    
    resampled_distribution = Counter(y_resampled)
    
    # Create a DataFrame for before and after sampling
    distribution_df = pd.DataFrame({
        'Embedding Type': embedding_type,
        'Class Distribution': list(original_distribution.keys()),
        'Before Sampling': [original_distribution[c] for c in original_distribution.keys()],
        'After Sampling': [resampled_distribution.get(c, 0) for c in original_distribution.keys()]
    })
    
    # print("\nClass Distribution Before and After Resampling:")
    # print(distribution_df)
    
    return X_resampled, y_resampled, distribution_df

# Dictionary to store resampled datasets
resampled_data = {}
distribution_results = []

embedding_types = ['Word2Vec', 'GloVe', 'Sentence Transformer', "TF-IDF", "BoW"]
datasets = {
    'train': (X_train_word2vec, X_train_glove, X_train_sentence_transformer, X_train_tfidf, X_train_bow,y_train),
    'test': (X_test_word2vec, X_test_glove, X_test_sentence_transformer, X_test_tfidf, X_test_bow, y_test),
    'val': (X_val_word2vec, X_val_glove, X_val_sentence_transformer, X_val_tfidf, X_val_bow, y_val)
}
            
# Apply resampling and store in dictionary
for dataset_type, (X_word2vec, X_glove, X_sentence, X_tfidf, X_bow, y) in datasets.items():
    print(f"\n📊 {dataset_type.capitalize()} Set Class Distribution Before Resampling: {Counter(y)}")
    
    for embedding, X_data in zip(embedding_types, [X_word2vec, X_glove, X_sentence, X_tfidf, X_bow]):
        X_resampled, y_resampled, distribution_df = apply_resampling(X_data, y, strategy = 'both', embedding_type = embedding)
        
        # Store in dictionary
        clean_embedding = embedding.replace("-", "").lower()  # Remove hyphen and make lowercase
        resampled_data[f'X_{dataset_type}_{clean_embedding}'] = X_resampled
        resampled_data[f'y_{dataset_type}'] = y_resampled
        
        # Store distribution results
        distribution_df['Dataset'] = dataset_type.capitalize()
        distribution_df['Embedding Type'] = embedding
        distribution_results.append(distribution_df)

# Combine all distributions into a DataFrame
final_resampling_df = pd.concat(distribution_results, ignore_index = True)

# Calculate overall totals before and after sampling (across all embeddings)
overall_total_before_sampling = final_resampling_df.groupby('Dataset')['Before Sampling'].sum().sum()
overall_total_after_sampling = final_resampling_df.groupby('Dataset')['After Sampling'].sum().sum()

# Print overall totals **before** displaying tables
print("\nOverall Total Samples Before Sampling:", int(overall_total_before_sampling/4))
print("Overall Total Samples After Sampling:", int(overall_total_after_sampling/4))

print("\n")
final_resampling_df.groupby(['Embedding Type', 'Dataset'])[['Before Sampling', 'After Sampling']].sum()

📊 Train Set Class Distribution Before Resampling: Counter({1: 185, 2: 43, 0: 22})

📊 Test Set Class Distribution Before Resampling: Counter({1: 62, 2: 14, 0: 8})

📊 Val Set Class Distribution Before Resampling: Counter({1: 62, 2: 14, 0: 8})

Overall Total Samples Before Sampling: 522
Overall Total Samples After Sampling: 1158

# Dictionary to store final DataFrames for each embedding
embedding_dfs = {}

# Generate separate DataFrames for each embedding type
for embedding in embedding_types:
    embedding_df = final_resampling_df[final_resampling_df['Embedding Type'] ==  embedding]
    
    # Pivot to structure as Accident Category (Class) vs. Train, Test, Validation
    table = embedding_df.pivot(index = 'Class Distribution', columns = 'Dataset', 
                               values = ['Before Sampling', 'After Sampling'])
    
    # Flatten MultiIndex columns for better readability
    table.columns = [f"{col[1]} {col[0]}" for col in table.columns]
    
    # Reset index for a proper DataFrame format
    table.reset_index(inplace = True)
    
    # Add a Total row
    total_row = table.sum(numeric_only = True)  # Sum all numerical columns
    #total_row['Class Distribution'] = 'Total'
    # Ensure the column can accept string values
    total_row['Class Distribution'] = total_row['Class Distribution'].astype(object)
    total_row['Class Distribution'] = 'Total'
    # Append the Total row
    table = pd.concat([table, pd.DataFrame(total_row).T], ignore_index = True)
    
    # Store DataFrame in dictionary
    embedding_dfs[embedding] = table

/tmp/ipykernel_19798/404156782.py:23: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value 'Total' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.
  total_row['Class Distribution'] = 'Total'
/tmp/ipykernel_19798/404156782.py:23: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value 'Total' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.
  total_row['Class Distribution'] = 'Total'
/tmp/ipykernel_19798/404156782.py:23: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value 'Total' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.
  total_row['Class Distribution'] = 'Total'
/tmp/ipykernel_19798/404156782.py:23: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value 'Total' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.
  total_row['Class Distribution'] = 'Total'
/tmp/ipykernel_19798/404156782.py:23: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise an error in a future version of pandas. Value 'Total' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.
  total_row['Class Distribution'] = 'Total'

print("\nAvailable keys in resampled_data:\n", resampled_data.keys())

Available keys in resampled_data:
 dict_keys(['X_train_word2vec', 'y_train', 'X_train_glove', 'X_train_sentence transformer', 'X_train_tfidf', 'X_train_bow', 'X_test_word2vec', 'y_test', 'X_test_glove', 'X_test_sentence transformer', 'X_test_tfidf', 'X_test_bow', 'X_val_word2vec', 'y_val', 'X_val_glove', 'X_val_sentence transformer', 'X_val_tfidf', 'X_val_bow'])

# Unpacking the resampled target variable sets and encoding them
le = LabelEncoder()
y_train_resampled = le.fit_transform(resampled_data['y_train'])
y_test_resampled = le.fit_transform(resampled_data['y_test'])
y_val_resampled = le.fit_transform(resampled_data['y_val'])

# Unpacking resampled Word2Vec variables
X_train_word2vec_resampled = resampled_data['X_train_word2vec']
X_test_word2vec_resampled = resampled_data['X_test_word2vec']
X_val_word2vec_resampled = resampled_data['X_val_word2vec']

# Access DataFrames
df_word2vec = embedding_dfs['Word2Vec']

# Print DataFrames
print("\nResampled Word2Vec DataFrame:")
df_word2vec

Resampled Word2Vec DataFrame:

# Unpacking resampled GloVe variables
X_train_glove_resampled = resampled_data['X_train_glove']
X_test_glove_resampled = resampled_data['X_test_glove']
X_val_glove_resampled = resampled_data['X_val_glove']

# Access DataFrames
df_glove = embedding_dfs['GloVe']

print("\nResampled GloVe DataFrame:")
df_glove

Resampled GloVe DataFrame:

# Unpacking resampled Sentence Transformer variables
X_train_sentence_transformer_resampled = resampled_data['X_train_sentence transformer']
X_test_sentence_transformer_resampled = resampled_data['X_test_sentence transformer']
X_val_sentence_transformer_resampled = resampled_data['X_val_sentence transformer']

# Access DataFrames
df_sentence_transformer = embedding_dfs['Sentence Transformer']

print("\nSentence Transformer DataFrame:")
df_sentence_transformer

Sentence Transformer DataFrame:

# Unpacking resampled Sentence Transformer variables
X_train_tfidf_resampled = resampled_data['X_train_tfidf']
X_test_tfidf_resampled = resampled_data['X_test_tfidf']
X_val_tfidf_resampled = resampled_data['X_val_tfidf']

# Access DataFrames
df_tfidf = embedding_dfs['TF-IDF']

print("\nTF-IDF DataFrame:")
df_tfidf

TF-IDF DataFrame:

# Unpacking resampled Sentence Transformer variables
X_train_bow_resampled = resampled_data['X_train_bow']
X_test_bow_resampled = resampled_data['X_test_bow']
X_val_bow_resampled = resampled_data['X_val_bow']

# Access DataFrames
df_bow = embedding_dfs['BoW']

print("\nBoW DataFrame:")
df_bow

BoW DataFrame:

# Creating Confusion Matrix
def plot_confusion_matrix(conf_matrix_train, conf_matrix_test, conf_matrix_val):

    conf_matrices = [conf_matrix_train, conf_matrix_test, conf_matrix_val]
    titles = ['Confusion Matrix for Training Set', 'Confusion Matrix for Testing Set', 'Confusion Matrix for Validation Set']
    
    fig, axes = plt.subplots(1, 3, figsize = (15, 5))
    
    # Labels for the confusion matrix
    labels = ['High', 'Low', 'Medium']
    
    for i, (conf_matrix, title) in enumerate(zip(conf_matrices, titles)):
        sns.heatmap(conf_matrix, annot = True, fmt = 'd', cmap = 'Blues', ax = axes[i], 
                    xticklabels = labels, yticklabels = labels, cbar = False)
        axes[i].set_title(title)
        axes[i].set_xlabel('Predicted Labels')
        axes[i].set_ylabel('Actual Labels')
        axes[i].tick_params(axis = 'x', labelsize = 12)
        axes[i].tick_params(axis = 'y', labelsize = 12)
    
    plt.subplots_adjust(wspace = 1.2) 
    plt.tight_layout()
    plt.show()

    print("Confusion Matrix for Final Training Set:\n", conf_matrix_train)
    print("Confusion Matrix for Final Testing Set:\n", conf_matrix_test)
    print("Confusion Matrix for Final Validation Set:\n", conf_matrix_val)

# Creating Evaluation Metrics for checking performance of the model
def evaluation_metrics(y_train, y_train_pred, y_test, y_test_pred, y_val, y_val_pred, conf_matrix_train, conf_matrix_test, conf_matrix_val):
    
    # Compute evaluation metrics for training set
    metrics_train = {
        "Accuracy": accuracy_score(y_train, y_train_pred),
        "Precision": precision_score(y_train, y_train_pred, average='weighted', zero_division=1),
        "Recall": recall_score(y_train, y_train_pred, average='weighted'),
        "F1-Score": f1_score(y_train, y_train_pred, average='weighted')
    }
    
    # Compute evaluation metrics for validation set
    metrics_val = {
        "Accuracy": accuracy_score(y_val, y_val_pred),
        "Precision": precision_score(y_val, y_val_pred, average='weighted', zero_division=1),
        "Recall": recall_score(y_val, y_val_pred, average='weighted'),
        "F1-Score": f1_score(y_val, y_val_pred, average='weighted')
    }
    
    # Compute evaluation metrics for testing set
    metrics_test = {
        "Accuracy": accuracy_score(y_test, y_test_pred),
        "Precision": precision_score(y_test, y_test_pred, average='weighted', zero_division=1),
        "Recall": recall_score(y_test, y_test_pred, average='weighted'),
        "F1-Score": f1_score(y_test, y_test_pred, average='weighted')
    }
    
    # Create DataFrame for tabular display
    metrics_df = pd.DataFrame([metrics_train, metrics_val, metrics_test], 
                              index=["Train", "Validation", "Test"])

    # Print the metrics table
    print("\nEvaluation Metrics in Tabular Format:")
    print(metrics_df)
    
    # Print Classification Reports
    print("\nClassification Reports:")
    print("\nTrain Classification Report:\n", classification_report(y_train, y_train_pred))
    print("\nValidation Classification Report:\n", classification_report(y_val, y_val_pred))
    print("\nTest Classification Report:\n", classification_report(y_test, y_test_pred))
    
    # Plot confusion matrix
    print("\nConfusion Matrices:")
    plot_confusion_matrix(conf_matrix_train, conf_matrix_test, conf_matrix_val)

def get_basic_model_metrics(model, X_train, y_train, X_test, y_test, X_val, y_val, embedding_name):
    """
    Performs a Model training along with printing the evaluation metrics
    
    Input Parameters:
    model: The model which we need to train
    X_train, y_train: Training sets of independent and target variables
    X_test, y_test: Testing sets of independent and target variables
    X_val, y_val: Validation sets of independent and target variables
    embedding_name: Type of word embedding used
    """
    model_name = model.__class__.__name__.replace('Classifier', '')
    
    # Training the model on the training set
    start_time = time.time()
    model.fit(X_train, y_train)
    end_time = time.time()

    time_to_fit = end_time - start_time

    print(f"\n Time taken to train the {model_name} model for {embedding_name} - {time_to_fit:.2f} seconds\n")
    
    print(" Model - ", model)

    #print("\n Evaluation Metrics - ")
    # Prediction
    y_train_pred = model.predict(X_train)
    y_test_pred = model.predict(X_test)
    y_val_pred = model.predict(X_val)
    
    conf_matrix_train = confusion_matrix(y_train, y_train_pred)
    conf_matrix_test = confusion_matrix(y_test, y_test_pred)
    conf_matrix_val = confusion_matrix(y_val, y_val_pred)

    evaluation_metrics(y_train, y_train_pred, y_test, y_test_pred, y_val, y_val_pred, 
                       conf_matrix_train, conf_matrix_test, conf_matrix_val)

# Function to compute different metrics to check performance of a classification model
def model_performance(model, X, y):
    """
    Function to compute different metrics to check classification model performance

    model: The current model
    X: independent variables
    y: target variable
    """

    # predicting using the independent variables
    pred = model.predict(X)
    
    if isinstance(pred[0], str):  # If predictions are in string format, encode them
        pred = le.transform(pred)

    accuracy = accuracy_score(y, pred)  # to compute Accuracy
    recall = recall_score(y, pred, average = 'weighted')  # to compute Recall
    precision = precision_score(y, pred, average = 'weighted', zero_division = 1)  # to compute Precision
    f1Score = f1_score(y, pred, average = 'weighted')  # to compute F1-score

    # creating a dataframe of metrics
    model_perf_df = pd.DataFrame(
        {
            "Accuracy": format(accuracy, '.4f'),
            "Recall": format(recall, '.4f'),
            "Precision": format(precision, '.4f'),
            "F1-Score": format(f1Score, '.4f'),
        },
        index = [0],
    )

    return model_perf_df

# Function to train and evaluate multiple models
def train_and_evaluate_models(models, X_train, y_train, X_test, y_test, X_val, y_val, embedding_name):
    """
    Train multiple models and evaluate them with print statements for progress tracking.
    
    models: Dictionary of models to train
    X_train, y_train: Training sets
    X_test, y_test: Testing sets
    X_val, y_val: Validation sets
    embedding_name: Name of the word embedding used
    
    Returns:
    model evaluation results
    """
    
    results = {}  # Store model performance
    results_list = []  # 📌 Added to store results for the table
    
    print("\n🔍 Starting model training and evaluation...\n")
    
    for i, (name, model) in enumerate(models.items(), start=1):
        print(f"\n   [{i}/{len(models)}] Training {name} model...")
            
        start_time = time.time()
        model.fit(X_train, y_train)
        end_time = time.time()
        
        training_time = end_time - start_time
        
        if training_time < 3600:
            print(f"\n   ✅ {name} training completed in : ", 
                  int(training_time / 60), "minute(s)", int(training_time % 60), "second(s)")
        else:
            print(f"\n   ✅ {name} training completed in : ", int(total_time // 3600), "hour(s)",
                  int(training_time / 60), "minute(s)", int(training_time % 60), "second(s)")

        # Evaluate the model on Train, Validation, and Test sets
        train_results = model_performance(model, X_train, y_train)
        val_results = model_performance(model, X_val, y_val)
        test_results = model_performance(model, X_test, y_test)

        # Store results
        results[name] = {
            "Train": train_results,
            "Validation": val_results,
            "Test": test_results
        }
        
        # 📌 Store results for tabular display
        results_list.append([
                                name, "Base",
                                train_results["Accuracy"].values[0], train_results["Precision"].values[0], 
                                train_results["Recall"].values[0], train_results["F1-Score"].values[0],
                                val_results["Accuracy"].values[0], val_results["Precision"].values[0], 
                                val_results["Recall"].values[0], val_results["F1-Score"].values[0],
                                test_results["Accuracy"].values[0], test_results["Precision"].values[0], 
                                test_results["Recall"].values[0], test_results["F1-Score"].values[0]
                            ])

        # 📌 Convert results into a DataFrame
        results_df = pd.DataFrame(results_list, columns=[
            "Model", "Type",
            "Train Acc", "Train Prec", "Train Recall", "Train F1",
            "Val Acc", "Val Prec", "Val Recall", "Val F1",
            "Test Acc", "Test Prec", "Test Recall", "Test F1"
        ])

        
    print("\n✅✅✅ All models trained and evaluated successfully!!!!!\n")
    return results, results_df

embedding_name = "Word2Vec"

# Defining models
models = {
    "RandomForest": RandomForestClassifier(),
    "DecisionTree": DecisionTreeClassifier(),
    "NaiveBayes": GaussianNB(),
    "AdaBoost": AdaBoostClassifier(),
    "GradientBoost": GradientBoostingClassifier(),
    "LogisticRegression": LogisticRegression(),
    "KNN": KNeighborsClassifier(),
    "SVM": SVC(),
    "XGBoost": XGBClassifier(use_label_encoder=False, eval_metric='mlogloss')
}

print("\n🔍 Starting Base Model training for all models (", embedding_name, " Embedding)...\n")

# Calling function to train and evaluate all models
base_results_word2vec, base_results_word2vec_df = train_and_evaluate_models(models, X_train_word2vec, y_train, X_test_word2vec,
                                                          y_test, X_val_word2vec, y_val, embedding_name)
base_results_word2vec_df

🔍 Starting Base Model training for all models ( Word2Vec  Embedding)...


🔍 Starting model training and evaluation...


   [1/9] Training RandomForest model...

   ✅ RandomForest training completed in :  0 minute(s) 0 second(s)

   [2/9] Training DecisionTree model...

   ✅ DecisionTree training completed in :  0 minute(s) 0 second(s)

   [3/9] Training NaiveBayes model...

   ✅ NaiveBayes training completed in :  0 minute(s) 0 second(s)

   [4/9] Training AdaBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/ensemble/_weight_boosting.py:527: FutureWarning: The SAMME.R algorithm (the default) is deprecated and will be removed in 1.6. Use the SAMME algorithm to circumvent this warning.
  warnings.warn(

   ✅ AdaBoost training completed in :  0 minute(s) 0 second(s)

   [5/9] Training GradientBoost model...

   ✅ GradientBoost training completed in :  0 minute(s) 3 second(s)

   [6/9] Training LogisticRegression model...

   ✅ LogisticRegression training completed in :  0 minute(s) 0 second(s)

   [7/9] Training KNN model...

   ✅ KNN training completed in :  0 minute(s) 0 second(s)

   [8/9] Training SVM model...

   ✅ SVM training completed in :  0 minute(s) 0 second(s)

   [9/9] Training XGBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/xgboost/core.py:158: UserWarning: [14:03:08] WARNING: /workspace/src/learner.cc:740: 
Parameters: { "use_label_encoder" } are not used.

  warnings.warn(smsg, UserWarning)

   ✅ XGBoost training completed in :  0 minute(s) 0 second(s)

✅✅✅ All models trained and evaluated successfully!!!!!

# Function to perform Grid Search on a model with progress tracking
def perform_grid_search(model, parameters, X_train, y_train, X_test, y_test, X_val, y_val, embedding_name):
    """
    Performs a Grid Search for a specific model along with time taken to fetch the best parameters.

    model: The model to perform Grid Search on
    parameters: The hyperparameter grid
    X_train: Training set of independent variables
    y_train: Training set of target variable
    embedding_name: Type of word embedding used
    """

    tuned_results_list = []  # Store tuned model results
    
    model_name = model.__class__.__name__.replace('Classifier', '')

    print(f"\n🔍 Starting Grid Search for {model_name} ({embedding_name})...\n")

    # Dynamically adjust cross-validation folds based on smallest class
    cv_folds = min(5, pd.Series(y_train).value_counts().min())

    # Initialize Grid Search
    grid_search = GridSearchCV(estimator=model, 
                               param_grid=parameters, 
                               cv=StratifiedKFold(n_splits=cv_folds, shuffle=True, random_state=42),  
                               n_jobs=-1, 
                               verbose=1,
                               error_score="raise"  # Prevent NaN scores
                              )

    # Training the model with Grid Search
    start_time_fit = time.time()
    grid_search.fit(X_train, y_train)
    end_time_fit = time.time()

    time_to_fit = end_time_fit - start_time_fit

    # Get best parameters
    best_parameters = grid_search.best_params_
    print(f"\n Best Parameters found for {model_name} ({embedding_name}) - \n{best_parameters}\n")

    # Print tuning time
    if time_to_fit < 3600:
        print(f"\n⏱️ {model_name} ({embedding_name}) Grid Search Time: {int(time_to_fit / 60)} min(s) {int(time_to_fit % 60)} sec(s)\n")
    else:
        print(f"\n⏱️ {model_name} ({embedding_name}) Grid Search Time: {int(time_to_fit // 3600)} hour(s) {int((time_to_fit % 3600) / 60)} min(s) {int(time_to_fit % 60)} sec(s)\n")

    print(f"\n Grid Search completed for {model_name} ({embedding_name})!\n")

    return grid_search

def evaluate_tuned_models(best_models, X_train, y_train, X_test, y_test, X_val, y_val, base_results_df):
    """
    Evaluates the best tuned models and prints a detailed comparison table.
    """
    tuned_results_list = []  # 📌 Store tuned model results

    print("\n Evaluating tuned models...\n")

    for i, (name, grid_search) in enumerate(best_models.items(), start=1):
        print(f" [{i}/{len(best_models)}] Evaluating best {name} model...")

        best_model = grid_search.best_estimator_  # Get the best model from GridSearchCV

        # Evaluate on Training, Validation, and Test sets
        train_results = model_performance(best_model, X_train, y_train)
        val_results = model_performance(best_model, X_val, y_val)
        test_results = model_performance(best_model, X_test, y_test)

        print(f" {name} model evaluation completed.\n")

        # 📌 Store results in a structured format for tabular display
        tuned_results_list.append([
            name, "Tuned",
            train_results["Accuracy"].values[0], train_results["Precision"].values[0], 
            train_results["Recall"].values[0], train_results["F1-Score"].values[0],
            val_results["Accuracy"].values[0], val_results["Precision"].values[0], 
            val_results["Recall"].values[0], val_results["F1-Score"].values[0],
            test_results["Accuracy"].values[0], test_results["Precision"].values[0], 
            test_results["Recall"].values[0], test_results["F1-Score"].values[0]
        ])

    # 📌 Convert tuned results into a DataFrame
    tuned_results_df = pd.DataFrame(tuned_results_list, columns=base_results_df.columns)

    # 📌 Merge Base and Tuned Results for Comparison
    comparison_df = pd.concat([base_results_df, tuned_results_df]).sort_values(by = ["Model", "Type"])
    comparison_df = comparison_df.sort_values(by=["Model", "Type"]).reset_index(drop = True)

    print("\n✅✅✅ All tuned models evaluated successfully!\n")
    print("\n Showing the Combined results for both Base and Tuned Models.....")
    
    # Return Comparison Table
    # display(comparison_df)
    return comparison_df  # Return DataFrame for further processing

# Defining hyperparameter grids
param_grids = {
    "RandomForest": {
        'n_estimators': np.arange(50, 150, 50),
        'criterion': ['gini', 'entropy'],
        'max_depth': np.arange(5, 30, 5), # maximum depth of the random forest
        'min_samples_split': [2, 5, 10], # these are the maximim number of samples required to split an internal node
        'min_samples_leaf': [1, 2, 4, 5], # minimum number of samples required to be at the tree node
        'max_features': [None, 'sqrt', 'log2'], # this sets the maximum number of features to be considered for splitting
        'max_leaf_nodes': [None, 10, 20], #this is the maximum number of leaf nodes for random forest
        'random_state': [42]
    },
    
    "DecisionTree": {
        'criterion': ['gini', 'entropy'],
        'max_depth': np.arange(5, 30, 5), # maximim depth of the decision tree
        'min_samples_split': [2, 5, 10], # these are the maximim number of samples required to split an internal node
        'min_samples_leaf': [1, 2, 5, 10], # minimum number of samples required to be at the tree node
        'max_features': [None, 'sqrt', 'log2'], # this sets the maximum number of features to be considered for splitting
        'max_leaf_nodes': [None, 5, 10, 20, 30], # this is the maximum number of leaf nodes for decision tree
        'random_state': [42]
    },
    
    "NaiveBayes": {},  # No hyperparameters for GaussianNB
    
    "AdaBoost": {
        'n_estimators': [50, 100, 200, 300], # number of estimators
        'learning_rate': [0.01, 0.1, 0.5, 1.0], # contribution of base estimator to the final model
        'algorithm': ['SAMME'],
        'random_state': [42]
    },
    
    "GradientBoost": {
        'n_estimators': [50, 100], # the number of boosting stages to be run
        'learning_rate': [0.01, 0.1, 0.2], # this will reduce the contribution of each tree by the specified amount
        'max_depth': [3, 5, 10], # the maximum depth of individual tree
        'min_samples_split': [2, 5, 10], # the minimum number of samples required to split an internal node. Helps control overfitting
        'min_samples_leaf': [1, 2, 4], # the minimum number of samples required to be at a leaf node. Helps control overfitting
        'max_features': ['sqrt', 'log2'], # the number of features to consider when looking for the best split
        'subsample': [0.8, 0.9, 1.0], # the fraction of samples to be used for fitting the individual base learners
        'random_state': [42]
    },
    
    "LogisticRegression": {
        'C': [0.01, 0.1, 1, 10],
        'solver': ['liblinear', 'lbfgs'],
    },
    
    "KNN": {
        'n_neighbors': [3, 5, 7, 9],
        'weights': ['uniform', 'distance'],
        'metric': ['euclidean', 'manhattan'],
    },
    
    "SVM": {
        'C': [0.1, 1, 10],  # Regularization parameter
        'kernel': ['linear', 'rbf', 'poly'],  # Kernel type
        'gamma': ['scale', 'auto'],  # Kernel coefficient
        'degree': [3, 4, 5],  # Degree of polynomial kernel functions (only relevant for 'poly' kernel)
        'coef0': [0.0, 0.5, 1.0]  # Independent term in kernel function (only relevant for 'poly' and 'sigmoid')
    },
    
    #"XGBoost": {
    #    'n_estimators': [50, 100, 200],
    #    'learning_rate': [0.01, 0.1, 0.2],
    #    'max_depth': [3, 5, 10],
    #    'random_state': [42]
    #}
}

# Perform Grid Search for all models
best_models_word2vec = {}

print("\n🔍 Starting hyperparameter tuning for all models (", embedding_name, " Embedding)...\n")

for i, (name, model) in enumerate(models.items(), start = 1):
    if name in param_grids and param_grids[name]:  # Skip models with no params (e.g., Naive Bayes)
        print(f"\n [{i}/{len(models)}] Started Hyperparameter tuning for {name}...")
        best_models_word2vec[name] = perform_grid_search(model, param_grids[name], X_train_word2vec, y_train, X_test_word2vec,
                                                y_test, X_val_word2vec, y_val, embedding_name)
        print("***********************************************************************")
    else:
        print(f"\n [{i}/{len(models)}] Skipped Hyperparameter tuning for {name} as there are no hyperparameters for this currently...")
        print("\n***********************************************************************")

print("\n\n✅✅✅ All models have been tuned successfully!\n")

🔍 Starting hyperparameter tuning for all models ( Word2Vec  Embedding)...


 [1/9] Started Hyperparameter tuning for RandomForest...

🔍 Starting Grid Search for RandomForest (Word2Vec)...

Fitting 5 folds for each of 2160 candidates, totalling 10800 fits

 Best Parameters found for RandomForest (Word2Vec) - 
{'criterion': 'gini', 'max_depth': 10, 'max_features': 'sqrt', 'max_leaf_nodes': 10, 'min_samples_leaf': 2, 'min_samples_split': 5, 'n_estimators': 50, 'random_state': 42}


⏱️ RandomForest (Word2Vec) Grid Search Time: 2 min(s) 12 sec(s)


 Grid Search completed for RandomForest (Word2Vec)!

***********************************************************************

 [2/9] Started Hyperparameter tuning for DecisionTree...

🔍 Starting Grid Search for DecisionTree (Word2Vec)...

Fitting 5 folds for each of 1800 candidates, totalling 9000 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for DecisionTree (Word2Vec) - 
{'criterion': 'gini', 'max_depth': 5, 'max_features': 'log2', 'max_leaf_nodes': 5, 'min_samples_leaf': 10, 'min_samples_split': 2, 'random_state': 42}


⏱️ DecisionTree (Word2Vec) Grid Search Time: 0 min(s) 2 sec(s)


 Grid Search completed for DecisionTree (Word2Vec)!

***********************************************************************

 [3/9] Skipped Hyperparameter tuning for NaiveBayes as there are no hyperparameters for this currently...

***********************************************************************

 [4/9] Started Hyperparameter tuning for AdaBoost...

🔍 Starting Grid Search for AdaBoost (Word2Vec)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for AdaBoost (Word2Vec) - 
{'algorithm': 'SAMME', 'learning_rate': 0.01, 'n_estimators': 100, 'random_state': 42}


⏱️ AdaBoost (Word2Vec) Grid Search Time: 0 min(s) 2 sec(s)


 Grid Search completed for AdaBoost (Word2Vec)!

***********************************************************************

 [5/9] Started Hyperparameter tuning for GradientBoost...

🔍 Starting Grid Search for GradientBoosting (Word2Vec)...

Fitting 5 folds for each of 972 candidates, totalling 4860 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for GradientBoosting (Word2Vec) - 
{'learning_rate': 0.01, 'max_depth': 3, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 50, 'random_state': 42, 'subsample': 0.8}


⏱️ GradientBoosting (Word2Vec) Grid Search Time: 0 min(s) 41 sec(s)


 Grid Search completed for GradientBoosting (Word2Vec)!

***********************************************************************

 [6/9] Started Hyperparameter tuning for LogisticRegression...

🔍 Starting Grid Search for LogisticRegression (Word2Vec)...

Fitting 5 folds for each of 8 candidates, totalling 40 fits

 Best Parameters found for LogisticRegression (Word2Vec) - 
{'C': 0.01, 'solver': 'liblinear'}


⏱️ LogisticRegression (Word2Vec) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for LogisticRegression (Word2Vec)!

***********************************************************************

 [7/9] Started Hyperparameter tuning for KNN...

🔍 Starting Grid Search for KNeighbors (Word2Vec)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for KNeighbors (Word2Vec) - 
{'metric': 'euclidean', 'n_neighbors': 9, 'weights': 'uniform'}


⏱️ KNeighbors (Word2Vec) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for KNeighbors (Word2Vec)!

***********************************************************************

 [8/9] Started Hyperparameter tuning for SVM...

🔍 Starting Grid Search for SVC (Word2Vec)...

Fitting 5 folds for each of 162 candidates, totalling 810 fits

 Best Parameters found for SVC (Word2Vec) - 
{'C': 0.1, 'coef0': 0.0, 'degree': 3, 'gamma': 'scale', 'kernel': 'linear'}


⏱️ SVC (Word2Vec) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for SVC (Word2Vec)!

***********************************************************************

 [9/9] Skipped Hyperparameter tuning for XGBoost as there are no hyperparameters for this currently...

***********************************************************************


✅✅✅ All models have been tuned successfully!

#Evaluating all models
tuned_word2vec_df = evaluate_tuned_models(best_models_word2vec, X_train_word2vec, y_train, X_test_word2vec,
                                 y_test, X_val_word2vec, y_val, base_results_word2vec_df)
tuned_word2vec_df

 Evaluating tuned models...

 [1/7] Evaluating best RandomForest model...
 RandomForest model evaluation completed.

 [2/7] Evaluating best DecisionTree model...
 DecisionTree model evaluation completed.

 [3/7] Evaluating best AdaBoost model...
 AdaBoost model evaluation completed.

 [4/7] Evaluating best GradientBoost model...
 GradientBoost model evaluation completed.

 [5/7] Evaluating best LogisticRegression model...
 LogisticRegression model evaluation completed.

 [6/7] Evaluating best KNN model...
 KNN model evaluation completed.

 [7/7] Evaluating best SVM model...
 SVM model evaluation completed.


✅✅✅ All tuned models evaluated successfully!


 Showing the Combined results for both Base and Tuned Models.....

print("\n🔍 Starting Base Model training with Resampled Data (", embedding_name, " Embedding)...\n")

# Calling function to train and evaluate all models
base_results_tuned_word2vec, base_results_tuned_word2vec_df = train_and_evaluate_models(models, X_train_word2vec_resampled, 
                                                                                        y_train_resampled, 
                                                                                        X_test_word2vec_resampled, 
                                                                                        y_test_resampled,
                                                                                        X_val_word2vec_resampled, 
                                                                                        y_val_resampled, embedding_name)
base_results_tuned_word2vec_df

🔍 Starting Base Model training with Resampled Data ( Word2Vec  Embedding)...


🔍 Starting model training and evaluation...


   [1/9] Training RandomForest model...

   ✅ RandomForest training completed in :  0 minute(s) 0 second(s)

   [2/9] Training DecisionTree model...

   ✅ DecisionTree training completed in :  0 minute(s) 0 second(s)

   [3/9] Training NaiveBayes model...

   ✅ NaiveBayes training completed in :  0 minute(s) 0 second(s)

   [4/9] Training AdaBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/ensemble/_weight_boosting.py:527: FutureWarning: The SAMME.R algorithm (the default) is deprecated and will be removed in 1.6. Use the SAMME algorithm to circumvent this warning.
  warnings.warn(

   ✅ AdaBoost training completed in :  0 minute(s) 0 second(s)

   [5/9] Training GradientBoost model...

   ✅ GradientBoost training completed in :  0 minute(s) 7 second(s)

   [6/9] Training LogisticRegression model...

   ✅ LogisticRegression training completed in :  0 minute(s) 0 second(s)

   [7/9] Training KNN model...

   ✅ KNN training completed in :  0 minute(s) 0 second(s)

   [8/9] Training SVM model...

   ✅ SVM training completed in :  0 minute(s) 0 second(s)

   [9/9] Training XGBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/xgboost/core.py:158: UserWarning: [14:06:17] WARNING: /workspace/src/learner.cc:740: 
Parameters: { "use_label_encoder" } are not used.

  warnings.warn(smsg, UserWarning)

   ✅ XGBoost training completed in :  0 minute(s) 0 second(s)

✅✅✅ All models trained and evaluated successfully!!!!!

# Perform Grid Search for all models
best_models_resampled_word2vec = {}

print("\n🔍 Starting hyperparameter tuning for all models with Resampled Data (", embedding_name, " Embedding)...\n")

for i, (name, model) in enumerate(models.items(), start = 1):
    if name in param_grids and param_grids[name]:  # Skip models with no params (e.g., Naive Bayes)
        print(f"\n    [{i}/{len(models)}] Started Hyperparameter tuning for {name}...")
        best_models_resampled_word2vec[name] = perform_grid_search(model, param_grids[name], X_train_word2vec_resampled, 
                                                               y_train_resampled, X_test_word2vec_resampled, y_test_resampled,
                                                               X_val_word2vec_resampled, y_val_resampled, embedding_name)
        print("***********************************************************************")
    else:
        print(f"\n [{i}/{len(models)}] Skipped Hyperparameter tuning for {name} as there are no hyperparameters for this...")
        print("\n***********************************************************************")

print("\n\n✅✅✅ All models have been tuned successfully!\n")

🔍 Starting hyperparameter tuning for all models with Resampled Data ( Word2Vec  Embedding)...


    [1/9] Started Hyperparameter tuning for RandomForest...

🔍 Starting Grid Search for RandomForest (Word2Vec)...

Fitting 5 folds for each of 2160 candidates, totalling 10800 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for RandomForest (Word2Vec) - 
{'criterion': 'gini', 'max_depth': 10, 'max_features': 'log2', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 100, 'random_state': 42}


⏱️ RandomForest (Word2Vec) Grid Search Time: 3 min(s) 16 sec(s)


 Grid Search completed for RandomForest (Word2Vec)!

***********************************************************************

    [2/9] Started Hyperparameter tuning for DecisionTree...

🔍 Starting Grid Search for DecisionTree (Word2Vec)...

Fitting 5 folds for each of 1800 candidates, totalling 9000 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for DecisionTree (Word2Vec) - 
{'criterion': 'entropy', 'max_depth': 15, 'max_features': 'sqrt', 'max_leaf_nodes': None, 'min_samples_leaf': 2, 'min_samples_split': 2, 'random_state': 42}


⏱️ DecisionTree (Word2Vec) Grid Search Time: 0 min(s) 4 sec(s)


 Grid Search completed for DecisionTree (Word2Vec)!

***********************************************************************

 [3/9] Skipped Hyperparameter tuning for NaiveBayes as there are no hyperparameters for this...

***********************************************************************

    [4/9] Started Hyperparameter tuning for AdaBoost...

🔍 Starting Grid Search for AdaBoost (Word2Vec)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for AdaBoost (Word2Vec) - 
{'algorithm': 'SAMME', 'learning_rate': 1.0, 'n_estimators': 200, 'random_state': 42}


⏱️ AdaBoost (Word2Vec) Grid Search Time: 0 min(s) 5 sec(s)


 Grid Search completed for AdaBoost (Word2Vec)!

***********************************************************************

    [5/9] Started Hyperparameter tuning for GradientBoost...

🔍 Starting Grid Search for GradientBoosting (Word2Vec)...

Fitting 5 folds for each of 972 candidates, totalling 4860 fits

 Best Parameters found for GradientBoosting (Word2Vec) - 
{'learning_rate': 0.2, 'max_depth': 10, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 100, 'random_state': 42, 'subsample': 0.8}


⏱️ GradientBoosting (Word2Vec) Grid Search Time: 0 min(s) 56 sec(s)


 Grid Search completed for GradientBoosting (Word2Vec)!

***********************************************************************

    [6/9] Started Hyperparameter tuning for LogisticRegression...

🔍 Starting Grid Search for LogisticRegression (Word2Vec)...

Fitting 5 folds for each of 8 candidates, totalling 40 fits

 Best Parameters found for LogisticRegression (Word2Vec) - 
{'C': 0.01, 'solver': 'lbfgs'}


⏱️ LogisticRegression (Word2Vec) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for LogisticRegression (Word2Vec)!

***********************************************************************

    [7/9] Started Hyperparameter tuning for KNN...

🔍 Starting Grid Search for KNeighbors (Word2Vec)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for KNeighbors (Word2Vec) - 
{'metric': 'euclidean', 'n_neighbors': 3, 'weights': 'distance'}


⏱️ KNeighbors (Word2Vec) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for KNeighbors (Word2Vec)!

***********************************************************************

    [8/9] Started Hyperparameter tuning for SVM...

🔍 Starting Grid Search for SVC (Word2Vec)...

Fitting 5 folds for each of 162 candidates, totalling 810 fits

 Best Parameters found for SVC (Word2Vec) - 
{'C': 10, 'coef0': 1.0, 'degree': 5, 'gamma': 'scale', 'kernel': 'poly'}


⏱️ SVC (Word2Vec) Grid Search Time: 0 min(s) 1 sec(s)


 Grid Search completed for SVC (Word2Vec)!

***********************************************************************

 [9/9] Skipped Hyperparameter tuning for XGBoost as there are no hyperparameters for this...

***********************************************************************


✅✅✅ All models have been tuned successfully!

#Evaluating all models
tuned_resampled_word2vec_df = evaluate_tuned_models(best_models_resampled_word2vec, X_train_word2vec_resampled, y_train_resampled, 
                                           X_test_word2vec_resampled, y_test_resampled, X_val_word2vec_resampled, 
                                           y_val_resampled, base_results_tuned_word2vec_df)
tuned_resampled_word2vec_df

 Evaluating tuned models...

 [1/7] Evaluating best RandomForest model...
 RandomForest model evaluation completed.

 [2/7] Evaluating best DecisionTree model...
 DecisionTree model evaluation completed.

 [3/7] Evaluating best AdaBoost model...
 AdaBoost model evaluation completed.

 [4/7] Evaluating best GradientBoost model...
 GradientBoost model evaluation completed.

 [5/7] Evaluating best LogisticRegression model...
 LogisticRegression model evaluation completed.

 [6/7] Evaluating best KNN model...
 KNN model evaluation completed.

 [7/7] Evaluating best SVM model...
 SVM model evaluation completed.


✅✅✅ All tuned models evaluated successfully!


 Showing the Combined results for both Base and Tuned Models.....

embedding_name = "GloVe"

print("\n🔍 Starting Base Model training for all models (", embedding_name, " Embedding)...\n")

# Calling function to train and evaluate all models
base_results_glove, base_results_glove_df = train_and_evaluate_models(models, X_train_glove, y_train, X_test_glove,
                                                          y_test, X_val_glove, y_val, embedding_name)
base_results_glove_df

🔍 Starting Base Model training for all models ( GloVe  Embedding)...


🔍 Starting model training and evaluation...


   [1/9] Training RandomForest model...

   ✅ RandomForest training completed in :  0 minute(s) 0 second(s)

   [2/9] Training DecisionTree model...

   ✅ DecisionTree training completed in :  0 minute(s) 0 second(s)

   [3/9] Training NaiveBayes model...

   ✅ NaiveBayes training completed in :  0 minute(s) 0 second(s)

   [4/9] Training AdaBoost model...

   ✅ AdaBoost training completed in :  0 minute(s) 0 second(s)

   [5/9] Training GradientBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/ensemble/_weight_boosting.py:527: FutureWarning: The SAMME.R algorithm (the default) is deprecated and will be removed in 1.6. Use the SAMME algorithm to circumvent this warning.
  warnings.warn(

   ✅ GradientBoost training completed in :  0 minute(s) 1 second(s)

   [6/9] Training LogisticRegression model...

   ✅ LogisticRegression training completed in :  0 minute(s) 0 second(s)

   [7/9] Training KNN model...

   ✅ KNN training completed in :  0 minute(s) 0 second(s)

   [8/9] Training SVM model...

   ✅ SVM training completed in :  0 minute(s) 0 second(s)

   [9/9] Training XGBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/xgboost/core.py:158: UserWarning: [14:10:45] WARNING: /workspace/src/learner.cc:740: 
Parameters: { "use_label_encoder" } are not used.

  warnings.warn(smsg, UserWarning)

   ✅ XGBoost training completed in :  0 minute(s) 0 second(s)

✅✅✅ All models trained and evaluated successfully!!!!!

# Perform Grid Search for all models
best_models_glove = {}

print("\n🔍 Starting hyperparameter tuning for all models (", embedding_name, " Embedding)...\n")

for i, (name, model) in enumerate(models.items(), start = 1):
    if name in param_grids and param_grids[name]:  # Skip models with no params (e.g., Naive Bayes)
        print(f"\n [{i}/{len(models)}] Started Hyperparameter tuning for {name}...")
        best_models_glove[name] = perform_grid_search(model, param_grids[name], X_train_glove, y_train, X_test_glove,
                                                y_test, X_val_glove, y_val, embedding_name)
        print("***********************************************************************")
    else:
        print(f"\n [{i}/{len(models)}] Skipped Hyperparameter tuning for {name} as there are no hyperparameters for this currently...")
        print("\n***********************************************************************")

print("\n\n✅✅✅ All models have been tuned successfully!\n")

🔍 Starting hyperparameter tuning for all models ( GloVe  Embedding)...


 [1/9] Started Hyperparameter tuning for RandomForest...

🔍 Starting Grid Search for RandomForest (GloVe)...

Fitting 5 folds for each of 2160 candidates, totalling 10800 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for RandomForest (GloVe) - 
{'criterion': 'gini', 'max_depth': 5, 'max_features': 'sqrt', 'max_leaf_nodes': None, 'min_samples_leaf': 2, 'min_samples_split': 5, 'n_estimators': 50, 'random_state': 42}


⏱️ RandomForest (GloVe) Grid Search Time: 2 min(s) 1 sec(s)


 Grid Search completed for RandomForest (GloVe)!

***********************************************************************

 [2/9] Started Hyperparameter tuning for DecisionTree...

🔍 Starting Grid Search for DecisionTree (GloVe)...

Fitting 5 folds for each of 1800 candidates, totalling 9000 fits

 Best Parameters found for DecisionTree (GloVe) - 
{'criterion': 'gini', 'max_depth': 5, 'max_features': 'sqrt', 'max_leaf_nodes': 5, 'min_samples_leaf': 10, 'min_samples_split': 2, 'random_state': 42}


⏱️ DecisionTree (GloVe) Grid Search Time: 0 min(s) 2 sec(s)


 Grid Search completed for DecisionTree (GloVe)!

***********************************************************************

 [3/9] Skipped Hyperparameter tuning for NaiveBayes as there are no hyperparameters for this currently...

***********************************************************************

 [4/9] Started Hyperparameter tuning for AdaBoost...

🔍 Starting Grid Search for AdaBoost (GloVe)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for AdaBoost (GloVe) - 
{'algorithm': 'SAMME', 'learning_rate': 0.01, 'n_estimators': 50, 'random_state': 42}


⏱️ AdaBoost (GloVe) Grid Search Time: 0 min(s) 1 sec(s)


 Grid Search completed for AdaBoost (GloVe)!

***********************************************************************

 [5/9] Started Hyperparameter tuning for GradientBoost...

🔍 Starting Grid Search for GradientBoosting (GloVe)...

Fitting 5 folds for each of 972 candidates, totalling 4860 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for GradientBoosting (GloVe) - 
{'learning_rate': 0.01, 'max_depth': 3, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 50, 'random_state': 42, 'subsample': 0.8}


⏱️ GradientBoosting (GloVe) Grid Search Time: 0 min(s) 36 sec(s)


 Grid Search completed for GradientBoosting (GloVe)!

***********************************************************************

 [6/9] Started Hyperparameter tuning for LogisticRegression...

🔍 Starting Grid Search for LogisticRegression (GloVe)...

Fitting 5 folds for each of 8 candidates, totalling 40 fits

 Best Parameters found for LogisticRegression (GloVe) - 
{'C': 0.01, 'solver': 'liblinear'}


⏱️ LogisticRegression (GloVe) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for LogisticRegression (GloVe)!

***********************************************************************

 [7/9] Started Hyperparameter tuning for KNN...

🔍 Starting Grid Search for KNeighbors (GloVe)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for KNeighbors (GloVe) - 
{'metric': 'euclidean', 'n_neighbors': 9, 'weights': 'uniform'}


⏱️ KNeighbors (GloVe) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for KNeighbors (GloVe)!

***********************************************************************

 [8/9] Started Hyperparameter tuning for SVM...

🔍 Starting Grid Search for SVC (GloVe)...

Fitting 5 folds for each of 162 candidates, totalling 810 fits

 Best Parameters found for SVC (GloVe) - 
{'C': 0.1, 'coef0': 0.0, 'degree': 3, 'gamma': 'scale', 'kernel': 'linear'}


⏱️ SVC (GloVe) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for SVC (GloVe)!

***********************************************************************

 [9/9] Skipped Hyperparameter tuning for XGBoost as there are no hyperparameters for this currently...

***********************************************************************


✅✅✅ All models have been tuned successfully!

#Evaluating all models
tuned_glove_df = evaluate_tuned_models(best_models_glove, X_train_glove, y_train, X_test_glove,
                                 y_test, X_val_glove, y_val, base_results_glove_df)
tuned_glove_df

 Evaluating tuned models...

 [1/7] Evaluating best RandomForest model...
 RandomForest model evaluation completed.

 [2/7] Evaluating best DecisionTree model...
 DecisionTree model evaluation completed.

 [3/7] Evaluating best AdaBoost model...
 AdaBoost model evaluation completed.

 [4/7] Evaluating best GradientBoost model...
 GradientBoost model evaluation completed.

 [5/7] Evaluating best LogisticRegression model...
 LogisticRegression model evaluation completed.

 [6/7] Evaluating best KNN model...
 KNN model evaluation completed.

 [7/7] Evaluating best SVM model...
 SVM model evaluation completed.


✅✅✅ All tuned models evaluated successfully!


 Showing the Combined results for both Base and Tuned Models.....

print("\n🔍 Starting Base Model training with Resampled Data (", embedding_name, " Embedding)...\n")

# Calling function to train and evaluate all models
base_results_tuned_glove, base_results_tuned_glove_df = train_and_evaluate_models(models, X_train_word2vec_resampled, y_train_resampled, 
                                                                      X_test_word2vec_resampled, y_test_resampled, 
                                                                      X_val_word2vec_resampled, y_val_resampled, embedding_name)
base_results_tuned_glove_df

🔍 Starting Base Model training with Resampled Data ( GloVe  Embedding)...


🔍 Starting model training and evaluation...


   [1/9] Training RandomForest model...

   ✅ RandomForest training completed in :  0 minute(s) 0 second(s)

   [2/9] Training DecisionTree model...

   ✅ DecisionTree training completed in :  0 minute(s) 0 second(s)

   [3/9] Training NaiveBayes model...

   ✅ NaiveBayes training completed in :  0 minute(s) 0 second(s)

   [4/9] Training AdaBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/ensemble/_weight_boosting.py:527: FutureWarning: The SAMME.R algorithm (the default) is deprecated and will be removed in 1.6. Use the SAMME algorithm to circumvent this warning.
  warnings.warn(

   ✅ AdaBoost training completed in :  0 minute(s) 0 second(s)

   [5/9] Training GradientBoost model...

   ✅ GradientBoost training completed in :  0 minute(s) 7 second(s)

   [6/9] Training LogisticRegression model...

   ✅ LogisticRegression training completed in :  0 minute(s) 0 second(s)

   [7/9] Training KNN model...

   ✅ KNN training completed in :  0 minute(s) 0 second(s)

   [8/9] Training SVM model...

   ✅ SVM training completed in :  0 minute(s) 0 second(s)

   [9/9] Training XGBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/xgboost/core.py:158: UserWarning: [14:13:36] WARNING: /workspace/src/learner.cc:740: 
Parameters: { "use_label_encoder" } are not used.

  warnings.warn(smsg, UserWarning)

   ✅ XGBoost training completed in :  0 minute(s) 0 second(s)

✅✅✅ All models trained and evaluated successfully!!!!!

# Perform Grid Search for all models
best_models_resampled_glove = {}

print("\n🔍 Starting hyperparameter tuning for all models with Resampled Data (", embedding_name, " Embedding)...\n")

for i, (name, model) in enumerate(models.items(), start = 1):
    if name in param_grids and param_grids[name]:  # Skip models with no params (e.g., Naive Bayes)
        print(f"\n    [{i}/{len(models)}] Started Hyperparameter tuning for {name}...")
        best_models_resampled_glove[name] = perform_grid_search(model, param_grids[name], X_train_glove_resampled, 
                                                               y_train_resampled, X_test_glove_resampled, y_test_resampled,
                                                               X_val_glove_resampled, y_val_resampled, embedding_name)
        print("***********************************************************************")
    else:
        print(f"\n [{i}/{len(models)}] Skipped Hyperparameter tuning for {name} as there are no hyperparameters for this currently...")
        print("\n***********************************************************************")

print("\n\n✅✅✅ All models have been tuned successfully!\n")

🔍 Starting hyperparameter tuning for all models with Resampled Data ( GloVe  Embedding)...


    [1/9] Started Hyperparameter tuning for RandomForest...

🔍 Starting Grid Search for RandomForest (GloVe)...

Fitting 5 folds for each of 2160 candidates, totalling 10800 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for RandomForest (GloVe) - 
{'criterion': 'gini', 'max_depth': 20, 'max_features': 'log2', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 100, 'random_state': 42}


⏱️ RandomForest (GloVe) Grid Search Time: 2 min(s) 26 sec(s)


 Grid Search completed for RandomForest (GloVe)!

***********************************************************************

    [2/9] Started Hyperparameter tuning for DecisionTree...

🔍 Starting Grid Search for DecisionTree (GloVe)...

Fitting 5 folds for each of 1800 candidates, totalling 9000 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for DecisionTree (GloVe) - 
{'criterion': 'entropy', 'max_depth': 15, 'max_features': 'log2', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 5, 'random_state': 42}


⏱️ DecisionTree (GloVe) Grid Search Time: 0 min(s) 2 sec(s)


 Grid Search completed for DecisionTree (GloVe)!

***********************************************************************

 [3/9] Skipped Hyperparameter tuning for NaiveBayes as there are no hyperparameters for this currently...

***********************************************************************

    [4/9] Started Hyperparameter tuning for AdaBoost...

🔍 Starting Grid Search for AdaBoost (GloVe)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for AdaBoost (GloVe) - 
{'algorithm': 'SAMME', 'learning_rate': 1.0, 'n_estimators': 200, 'random_state': 42}


⏱️ AdaBoost (GloVe) Grid Search Time: 0 min(s) 3 sec(s)


 Grid Search completed for AdaBoost (GloVe)!

***********************************************************************

    [5/9] Started Hyperparameter tuning for GradientBoost...

🔍 Starting Grid Search for GradientBoosting (GloVe)...

Fitting 5 folds for each of 972 candidates, totalling 4860 fits

 Best Parameters found for GradientBoosting (GloVe) - 
{'learning_rate': 0.1, 'max_depth': 10, 'max_features': 'log2', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 100, 'random_state': 42, 'subsample': 0.9}


⏱️ GradientBoosting (GloVe) Grid Search Time: 0 min(s) 51 sec(s)


 Grid Search completed for GradientBoosting (GloVe)!

***********************************************************************

    [6/9] Started Hyperparameter tuning for LogisticRegression...

🔍 Starting Grid Search for LogisticRegression (GloVe)...

Fitting 5 folds for each of 8 candidates, totalling 40 fits

 Best Parameters found for LogisticRegression (GloVe) - 
{'C': 10, 'solver': 'lbfgs'}


⏱️ LogisticRegression (GloVe) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for LogisticRegression (GloVe)!

***********************************************************************

    [7/9] Started Hyperparameter tuning for KNN...

🔍 Starting Grid Search for KNeighbors (GloVe)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for KNeighbors (GloVe) - 
{'metric': 'euclidean', 'n_neighbors': 3, 'weights': 'distance'}


⏱️ KNeighbors (GloVe) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for KNeighbors (GloVe)!

***********************************************************************

    [8/9] Started Hyperparameter tuning for SVM...

🔍 Starting Grid Search for SVC (GloVe)...

Fitting 5 folds for each of 162 candidates, totalling 810 fits

/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/linear_model/_logistic.py:469: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(
/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/linear_model/_logistic.py:469: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(
/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/linear_model/_logistic.py:469: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(
/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/linear_model/_logistic.py:469: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(

 Best Parameters found for SVC (GloVe) - 
{'C': 10, 'coef0': 0.5, 'degree': 5, 'gamma': 'scale', 'kernel': 'poly'}


⏱️ SVC (GloVe) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for SVC (GloVe)!

***********************************************************************

 [9/9] Skipped Hyperparameter tuning for XGBoost as there are no hyperparameters for this currently...

***********************************************************************


✅✅✅ All models have been tuned successfully!

#Evaluating all models
tuned_resampled_glove_df = evaluate_tuned_models(best_models_resampled_glove, X_train_glove_resampled, y_train_resampled, 
                                           X_test_glove_resampled, y_test_resampled, X_val_glove_resampled, 
                                           y_val_resampled, base_results_tuned_glove_df)
tuned_resampled_glove_df

 Evaluating tuned models...

 [1/7] Evaluating best RandomForest model...
 RandomForest model evaluation completed.

 [2/7] Evaluating best DecisionTree model...
 DecisionTree model evaluation completed.

 [3/7] Evaluating best AdaBoost model...
 AdaBoost model evaluation completed.

 [4/7] Evaluating best GradientBoost model...
 GradientBoost model evaluation completed.

 [5/7] Evaluating best LogisticRegression model...
 LogisticRegression model evaluation completed.

 [6/7] Evaluating best KNN model...
 KNN model evaluation completed.

 [7/7] Evaluating best SVM model...
 SVM model evaluation completed.


✅✅✅ All tuned models evaluated successfully!


 Showing the Combined results for both Base and Tuned Models.....

embedding_name = "Sentence Transformer"

print("\n🔍 Starting Base Model training for all models (", embedding_name, " Embedding)...\n")

# Calling function to train and evaluate all models
base_results_sent_trans, base_results_sent_trans_df = train_and_evaluate_models(models, X_train_sentence_transformer, y_train, 
                                                          X_test_sentence_transformer, y_test, X_val_sentence_transformer, 
                                                          y_val, embedding_name)
base_results_sent_trans_df

🔍 Starting Base Model training for all models ( Sentence Transformer  Embedding)...


🔍 Starting model training and evaluation...


   [1/9] Training RandomForest model...

   ✅ RandomForest training completed in :  0 minute(s) 0 second(s)

   [2/9] Training DecisionTree model...

   ✅ DecisionTree training completed in :  0 minute(s) 0 second(s)

   [3/9] Training NaiveBayes model...

   ✅ NaiveBayes training completed in :  0 minute(s) 0 second(s)

   [4/9] Training AdaBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/ensemble/_weight_boosting.py:527: FutureWarning: The SAMME.R algorithm (the default) is deprecated and will be removed in 1.6. Use the SAMME algorithm to circumvent this warning.
  warnings.warn(

   ✅ AdaBoost training completed in :  0 minute(s) 0 second(s)

   [5/9] Training GradientBoost model...

   ✅ GradientBoost training completed in :  0 minute(s) 6 second(s)

   [6/9] Training LogisticRegression model...

   ✅ LogisticRegression training completed in :  0 minute(s) 0 second(s)

   [7/9] Training KNN model...

   ✅ KNN training completed in :  0 minute(s) 0 second(s)

   [8/9] Training SVM model...

   ✅ SVM training completed in :  0 minute(s) 0 second(s)

   [9/9] Training XGBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/xgboost/core.py:158: UserWarning: [14:17:09] WARNING: /workspace/src/learner.cc:740: 
Parameters: { "use_label_encoder" } are not used.

  warnings.warn(smsg, UserWarning)

   ✅ XGBoost training completed in :  0 minute(s) 1 second(s)

✅✅✅ All models trained and evaluated successfully!!!!!

# Perform Grid Search for all models
best_models_sent_trans = {}

print("\n🔍 Starting hyperparameter tuning for all models (", embedding_name, " Embedding)...\n")

for i, (name, model) in enumerate(models.items(), start = 1):
    if name in param_grids and param_grids[name]:  # Skip models with no params (e.g., Naive Bayes)
        print(f"\n [{i}/{len(models)}] Started Hyperparameter tuning for {name}...")
        best_models_sent_trans[name] = perform_grid_search(model, param_grids[name], X_train_sentence_transformer, y_train, 
                                                X_test_sentence_transformer, y_test, X_val_sentence_transformer, 
                                                y_val, embedding_name)
        print("***********************************************************************")
    else:
        print(f"\n [{i}/{len(models)}] Skipped Hyperparameter tuning for {name} as there are no hyperparameters for this currently...")
        print("\n***********************************************************************")

print("\n\n✅✅✅ All models have been tuned successfully!\n")

🔍 Starting hyperparameter tuning for all models ( Sentence Transformer  Embedding)...


 [1/9] Started Hyperparameter tuning for RandomForest...

🔍 Starting Grid Search for RandomForest (Sentence Transformer)...

Fitting 5 folds for each of 2160 candidates, totalling 10800 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for RandomForest (Sentence Transformer) - 
{'criterion': 'entropy', 'max_depth': 5, 'max_features': 'sqrt', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 10, 'n_estimators': 50, 'random_state': 42}


⏱️ RandomForest (Sentence Transformer) Grid Search Time: 2 min(s) 46 sec(s)


 Grid Search completed for RandomForest (Sentence Transformer)!

***********************************************************************

 [2/9] Started Hyperparameter tuning for DecisionTree...

🔍 Starting Grid Search for DecisionTree (Sentence Transformer)...

Fitting 5 folds for each of 1800 candidates, totalling 9000 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for DecisionTree (Sentence Transformer) - 
{'criterion': 'gini', 'max_depth': 5, 'max_features': 'log2', 'max_leaf_nodes': 5, 'min_samples_leaf': 1, 'min_samples_split': 10, 'random_state': 42}


⏱️ DecisionTree (Sentence Transformer) Grid Search Time: 0 min(s) 3 sec(s)


 Grid Search completed for DecisionTree (Sentence Transformer)!

***********************************************************************

 [3/9] Skipped Hyperparameter tuning for NaiveBayes as there are no hyperparameters for this currently...

***********************************************************************

 [4/9] Started Hyperparameter tuning for AdaBoost...

🔍 Starting Grid Search for AdaBoost (Sentence Transformer)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for AdaBoost (Sentence Transformer) - 
{'algorithm': 'SAMME', 'learning_rate': 0.1, 'n_estimators': 100, 'random_state': 42}


⏱️ AdaBoost (Sentence Transformer) Grid Search Time: 0 min(s) 3 sec(s)


 Grid Search completed for AdaBoost (Sentence Transformer)!

***********************************************************************

 [5/9] Started Hyperparameter tuning for GradientBoost...

🔍 Starting Grid Search for GradientBoosting (Sentence Transformer)...

Fitting 5 folds for each of 972 candidates, totalling 4860 fits

 Best Parameters found for GradientBoosting (Sentence Transformer) - 
{'learning_rate': 0.1, 'max_depth': 3, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 5, 'n_estimators': 50, 'random_state': 42, 'subsample': 0.8}


⏱️ GradientBoosting (Sentence Transformer) Grid Search Time: 0 min(s) 43 sec(s)


 Grid Search completed for GradientBoosting (Sentence Transformer)!

***********************************************************************

 [6/9] Started Hyperparameter tuning for LogisticRegression...

🔍 Starting Grid Search for LogisticRegression (Sentence Transformer)...

Fitting 5 folds for each of 8 candidates, totalling 40 fits

 Best Parameters found for LogisticRegression (Sentence Transformer) - 
{'C': 0.01, 'solver': 'liblinear'}


⏱️ LogisticRegression (Sentence Transformer) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for LogisticRegression (Sentence Transformer)!

***********************************************************************

 [7/9] Started Hyperparameter tuning for KNN...

🔍 Starting Grid Search for KNeighbors (Sentence Transformer)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for KNeighbors (Sentence Transformer) - 
{'metric': 'euclidean', 'n_neighbors': 9, 'weights': 'uniform'}


⏱️ KNeighbors (Sentence Transformer) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for KNeighbors (Sentence Transformer)!

***********************************************************************

 [8/9] Started Hyperparameter tuning for SVM...

🔍 Starting Grid Search for SVC (Sentence Transformer)...

Fitting 5 folds for each of 162 candidates, totalling 810 fits

 Best Parameters found for SVC (Sentence Transformer) - 
{'C': 0.1, 'coef0': 0.0, 'degree': 3, 'gamma': 'scale', 'kernel': 'linear'}


⏱️ SVC (Sentence Transformer) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for SVC (Sentence Transformer)!

***********************************************************************

 [9/9] Skipped Hyperparameter tuning for XGBoost as there are no hyperparameters for this currently...

***********************************************************************


✅✅✅ All models have been tuned successfully!

#Evaluating all models
tuned_sent_trans_df = evaluate_tuned_models(best_models_sent_trans, X_train_sentence_transformer, y_train, X_test_sentence_transformer,
                                 y_test, X_val_sentence_transformer, y_val, base_results_sent_trans_df)
tuned_sent_trans_df

 Evaluating tuned models...

 [1/7] Evaluating best RandomForest model...
 RandomForest model evaluation completed.

 [2/7] Evaluating best DecisionTree model...
 DecisionTree model evaluation completed.

 [3/7] Evaluating best AdaBoost model...
 AdaBoost model evaluation completed.

 [4/7] Evaluating best GradientBoost model...
 GradientBoost model evaluation completed.

 [5/7] Evaluating best LogisticRegression model...
 LogisticRegression model evaluation completed.

 [6/7] Evaluating best KNN model...
 KNN model evaluation completed.

 [7/7] Evaluating best SVM model...
 SVM model evaluation completed.


✅✅✅ All tuned models evaluated successfully!


 Showing the Combined results for both Base and Tuned Models.....

print("\n🔍 Starting Base Model training with Resampled Data (", embedding_name, " Embedding)...\n")

# Calling function to train and evaluate all models
base_results_tuned_sent_trans, base_results_tuned_sent_trans_df = train_and_evaluate_models(models, X_train_sentence_transformer_resampled, 
                                                                      y_train_resampled, X_test_sentence_transformer_resampled, 
                                                                      y_test_resampled, X_val_sentence_transformer_resampled, 
                                                                      y_val_resampled, embedding_name)
base_results_tuned_sent_trans_df

🔍 Starting Base Model training with Resampled Data ( Sentence Transformer  Embedding)...


🔍 Starting model training and evaluation...


   [1/9] Training RandomForest model...

   ✅ RandomForest training completed in :  0 minute(s) 0 second(s)

   [2/9] Training DecisionTree model...

   ✅ DecisionTree training completed in :  0 minute(s) 0 second(s)

   [3/9] Training NaiveBayes model...

   ✅ NaiveBayes training completed in :  0 minute(s) 0 second(s)

   [4/9] Training AdaBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/ensemble/_weight_boosting.py:527: FutureWarning: The SAMME.R algorithm (the default) is deprecated and will be removed in 1.6. Use the SAMME algorithm to circumvent this warning.
  warnings.warn(

   ✅ AdaBoost training completed in :  0 minute(s) 1 second(s)

   [5/9] Training GradientBoost model...

   ✅ GradientBoost training completed in :  0 minute(s) 15 second(s)

   [6/9] Training LogisticRegression model...

   ✅ LogisticRegression training completed in :  0 minute(s) 0 second(s)

   [7/9] Training KNN model...

   ✅ KNN training completed in :  0 minute(s) 0 second(s)

   [8/9] Training SVM model...

   ✅ SVM training completed in :  0 minute(s) 0 second(s)

   [9/9] Training XGBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/xgboost/core.py:158: UserWarning: [14:21:06] WARNING: /workspace/src/learner.cc:740: 
Parameters: { "use_label_encoder" } are not used.

  warnings.warn(smsg, UserWarning)

   ✅ XGBoost training completed in :  0 minute(s) 1 second(s)

✅✅✅ All models trained and evaluated successfully!!!!!

# Perform Grid Search for all models
best_models_resampled_sent_trans = {}

print("\n🔍 Starting hyperparameter tuning for all models with Resampled Data (", embedding_name, " Embedding)...\n")

for i, (name, model) in enumerate(models.items(), start = 1):
    if name in param_grids and param_grids[name]:  # Skip models with no params (e.g., Naive Bayes)
        print(f"\n    [{i}/{len(models)}] Started Hyperparameter tuning for {name}...")
        best_models_resampled_sent_trans[name] = perform_grid_search(model, param_grids[name], X_train_sentence_transformer_resampled,
                                                               y_train_resampled, X_test_sentence_transformer_resampled, 
                                                               y_test_resampled, X_val_sentence_transformer_resampled, 
                                                               y_val_resampled, embedding_name)
        print("***********************************************************************")
    else:
        print(f"\n [{i}/{len(models)}] Skipped Hyperparameter tuning for {name} as there are no hyperparameters for this...")
        print("\n***********************************************************************")

print("\n\n✅✅✅ All models have been tuned successfully!\n")

🔍 Starting hyperparameter tuning for all models with Resampled Data ( Sentence Transformer  Embedding)...


    [1/9] Started Hyperparameter tuning for RandomForest...

🔍 Starting Grid Search for RandomForest (Sentence Transformer)...

Fitting 5 folds for each of 2160 candidates, totalling 10800 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for RandomForest (Sentence Transformer) - 
{'criterion': 'entropy', 'max_depth': 10, 'max_features': 'log2', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 5, 'n_estimators': 100, 'random_state': 42}


⏱️ RandomForest (Sentence Transformer) Grid Search Time: 4 min(s) 59 sec(s)


 Grid Search completed for RandomForest (Sentence Transformer)!

***********************************************************************

    [2/9] Started Hyperparameter tuning for DecisionTree...

🔍 Starting Grid Search for DecisionTree (Sentence Transformer)...

Fitting 5 folds for each of 1800 candidates, totalling 9000 fits

 Best Parameters found for DecisionTree (Sentence Transformer) - 
{'criterion': 'gini', 'max_depth': 10, 'max_features': None, 'max_leaf_nodes': 30, 'min_samples_leaf': 2, 'min_samples_split': 2, 'random_state': 42}


⏱️ DecisionTree (Sentence Transformer) Grid Search Time: 0 min(s) 8 sec(s)


 Grid Search completed for DecisionTree (Sentence Transformer)!

***********************************************************************

 [3/9] Skipped Hyperparameter tuning for NaiveBayes as there are no hyperparameters for this...

***********************************************************************

    [4/9] Started Hyperparameter tuning for AdaBoost...

🔍 Starting Grid Search for AdaBoost (Sentence Transformer)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for AdaBoost (Sentence Transformer) - 
{'algorithm': 'SAMME', 'learning_rate': 1.0, 'n_estimators': 200, 'random_state': 42}


⏱️ AdaBoost (Sentence Transformer) Grid Search Time: 0 min(s) 10 sec(s)


 Grid Search completed for AdaBoost (Sentence Transformer)!

***********************************************************************

    [5/9] Started Hyperparameter tuning for GradientBoost...

🔍 Starting Grid Search for GradientBoosting (Sentence Transformer)...

Fitting 5 folds for each of 972 candidates, totalling 4860 fits

 Best Parameters found for GradientBoosting (Sentence Transformer) - 
{'learning_rate': 0.01, 'max_depth': 10, 'max_features': 'log2', 'min_samples_leaf': 1, 'min_samples_split': 5, 'n_estimators': 100, 'random_state': 42, 'subsample': 1.0}


⏱️ GradientBoosting (Sentence Transformer) Grid Search Time: 1 min(s) 2 sec(s)


 Grid Search completed for GradientBoosting (Sentence Transformer)!

***********************************************************************

    [6/9] Started Hyperparameter tuning for LogisticRegression...

🔍 Starting Grid Search for LogisticRegression (Sentence Transformer)...

Fitting 5 folds for each of 8 candidates, totalling 40 fits

 Best Parameters found for LogisticRegression (Sentence Transformer) - 
{'C': 10, 'solver': 'lbfgs'}


⏱️ LogisticRegression (Sentence Transformer) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for LogisticRegression (Sentence Transformer)!

***********************************************************************

    [7/9] Started Hyperparameter tuning for KNN...

🔍 Starting Grid Search for KNeighbors (Sentence Transformer)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for KNeighbors (Sentence Transformer) - 
{'metric': 'manhattan', 'n_neighbors': 3, 'weights': 'distance'}


⏱️ KNeighbors (Sentence Transformer) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for KNeighbors (Sentence Transformer)!

***********************************************************************

    [8/9] Started Hyperparameter tuning for SVM...

🔍 Starting Grid Search for SVC (Sentence Transformer)...

Fitting 5 folds for each of 162 candidates, totalling 810 fits

 Best Parameters found for SVC (Sentence Transformer) - 
{'C': 1, 'coef0': 0.0, 'degree': 5, 'gamma': 'scale', 'kernel': 'poly'}


⏱️ SVC (Sentence Transformer) Grid Search Time: 0 min(s) 1 sec(s)


 Grid Search completed for SVC (Sentence Transformer)!

***********************************************************************

 [9/9] Skipped Hyperparameter tuning for XGBoost as there are no hyperparameters for this...

***********************************************************************


✅✅✅ All models have been tuned successfully!

#Evaluating all models
tuned_resampled_sent_trans_df = evaluate_tuned_models(best_models_resampled_sent_trans, X_train_sentence_transformer_resampled, 
                                                      y_train_resampled, X_test_sentence_transformer_resampled, y_test_resampled, 
                                                      X_val_sentence_transformer_resampled, y_val_resampled, base_results_tuned_sent_trans_df)
tuned_resampled_sent_trans_df

 Evaluating tuned models...

 [1/7] Evaluating best RandomForest model...
 RandomForest model evaluation completed.

 [2/7] Evaluating best DecisionTree model...
 DecisionTree model evaluation completed.

 [3/7] Evaluating best AdaBoost model...
 AdaBoost model evaluation completed.

 [4/7] Evaluating best GradientBoost model...
 GradientBoost model evaluation completed.

 [5/7] Evaluating best LogisticRegression model...
 LogisticRegression model evaluation completed.

 [6/7] Evaluating best KNN model...
 KNN model evaluation completed.

 [7/7] Evaluating best SVM model...
 SVM model evaluation completed.


✅✅✅ All tuned models evaluated successfully!


 Showing the Combined results for both Base and Tuned Models.....

embedding_name = "TF-IDF"

print("\n🔍 Starting Base Model training for all models (", embedding_name, " Embedding)...\n")

# Calling function to train and evaluate all models
base_results_tfidf, base_results_tfidf_df = train_and_evaluate_models(models, X_train_tfidf, y_train, 
                                                          X_test_tfidf, y_test, X_val_tfidf, y_val, embedding_name)
base_results_tfidf_df

🔍 Starting Base Model training for all models ( TF-IDF  Embedding)...


🔍 Starting model training and evaluation...


   [1/9] Training RandomForest model...

   ✅ RandomForest training completed in :  0 minute(s) 0 second(s)

   [2/9] Training DecisionTree model...

   ✅ DecisionTree training completed in :  0 minute(s) 0 second(s)

   [3/9] Training NaiveBayes model...

   ✅ NaiveBayes training completed in :  0 minute(s) 0 second(s)

   [4/9] Training AdaBoost model...

   ✅ AdaBoost training completed in :  0 minute(s) 0 second(s)

   [5/9] Training GradientBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/ensemble/_weight_boosting.py:527: FutureWarning: The SAMME.R algorithm (the default) is deprecated and will be removed in 1.6. Use the SAMME algorithm to circumvent this warning.
  warnings.warn(

   ✅ GradientBoost training completed in :  0 minute(s) 1 second(s)

   [6/9] Training LogisticRegression model...

   ✅ LogisticRegression training completed in :  0 minute(s) 0 second(s)

   [7/9] Training KNN model...

   ✅ KNN training completed in :  0 minute(s) 0 second(s)

   [8/9] Training SVM model...

   ✅ SVM training completed in :  0 minute(s) 0 second(s)

   [9/9] Training XGBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/xgboost/core.py:158: UserWarning: [14:27:31] WARNING: /workspace/src/learner.cc:740: 
Parameters: { "use_label_encoder" } are not used.

  warnings.warn(smsg, UserWarning)

   ✅ XGBoost training completed in :  0 minute(s) 0 second(s)

✅✅✅ All models trained and evaluated successfully!!!!!

# Perform Grid Search for all models
best_models_tfidf = {}

print("\n🔍 Starting hyperparameter tuning for all models (", embedding_name, " Embedding)...\n")

for i, (name, model) in enumerate(models.items(), start = 1):
    if name in param_grids and param_grids[name]:  # Skip models with no params (e.g., Naive Bayes)
        print(f"\n [{i}/{len(models)}] Started Hyperparameter tuning for {name}...")
        best_models_tfidf[name] = perform_grid_search(model, param_grids[name], X_train_tfidf, y_train, 
                                                X_test_tfidf, y_test, X_val_tfidf, y_val, embedding_name)
        print("***********************************************************************")
    else:
        print(f"\n [{i}/{len(models)}] Skipped Hyperparameter tuning for {name} as there are no hyperparameters for this currently...")
        print("\n***********************************************************************")

print("\n\n✅✅✅ All models have been tuned successfully!\n")

🔍 Starting hyperparameter tuning for all models ( TF-IDF  Embedding)...


 [1/9] Started Hyperparameter tuning for RandomForest...

🔍 Starting Grid Search for RandomForest (TF-IDF)...

Fitting 5 folds for each of 2160 candidates, totalling 10800 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for RandomForest (TF-IDF) - 
{'criterion': 'gini', 'max_depth': 5, 'max_features': None, 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 50, 'random_state': 42}


⏱️ RandomForest (TF-IDF) Grid Search Time: 1 min(s) 53 sec(s)


 Grid Search completed for RandomForest (TF-IDF)!

***********************************************************************

 [2/9] Started Hyperparameter tuning for DecisionTree...

🔍 Starting Grid Search for DecisionTree (TF-IDF)...

Fitting 5 folds for each of 1800 candidates, totalling 9000 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for DecisionTree (TF-IDF) - 
{'criterion': 'gini', 'max_depth': 5, 'max_features': 'sqrt', 'max_leaf_nodes': None, 'min_samples_leaf': 10, 'min_samples_split': 2, 'random_state': 42}


⏱️ DecisionTree (TF-IDF) Grid Search Time: 0 min(s) 2 sec(s)


 Grid Search completed for DecisionTree (TF-IDF)!

***********************************************************************

 [3/9] Skipped Hyperparameter tuning for NaiveBayes as there are no hyperparameters for this currently...

***********************************************************************

 [4/9] Started Hyperparameter tuning for AdaBoost...

🔍 Starting Grid Search for AdaBoost (TF-IDF)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for AdaBoost (TF-IDF) - 
{'algorithm': 'SAMME', 'learning_rate': 0.01, 'n_estimators': 50, 'random_state': 42}


⏱️ AdaBoost (TF-IDF) Grid Search Time: 0 min(s) 1 sec(s)


 Grid Search completed for AdaBoost (TF-IDF)!

***********************************************************************

 [5/9] Started Hyperparameter tuning for GradientBoost...

🔍 Starting Grid Search for GradientBoosting (TF-IDF)...

Fitting 5 folds for each of 972 candidates, totalling 4860 fits

 Best Parameters found for GradientBoosting (TF-IDF) - 
{'learning_rate': 0.2, 'max_depth': 5, 'max_features': 'log2', 'min_samples_leaf': 4, 'min_samples_split': 10, 'n_estimators': 100, 'random_state': 42, 'subsample': 0.9}


⏱️ GradientBoosting (TF-IDF) Grid Search Time: 0 min(s) 26 sec(s)


 Grid Search completed for GradientBoosting (TF-IDF)!

***********************************************************************

 [6/9] Started Hyperparameter tuning for LogisticRegression...

🔍 Starting Grid Search for LogisticRegression (TF-IDF)...

Fitting 5 folds for each of 8 candidates, totalling 40 fits

 Best Parameters found for LogisticRegression (TF-IDF) - 
{'C': 0.01, 'solver': 'liblinear'}


⏱️ LogisticRegression (TF-IDF) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for LogisticRegression (TF-IDF)!

***********************************************************************

 [7/9] Started Hyperparameter tuning for KNN...

🔍 Starting Grid Search for KNeighbors (TF-IDF)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for KNeighbors (TF-IDF) - 
{'metric': 'manhattan', 'n_neighbors': 9, 'weights': 'uniform'}


⏱️ KNeighbors (TF-IDF) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for KNeighbors (TF-IDF)!

***********************************************************************

 [8/9] Started Hyperparameter tuning for SVM...

🔍 Starting Grid Search for SVC (TF-IDF)...

Fitting 5 folds for each of 162 candidates, totalling 810 fits

 Best Parameters found for SVC (TF-IDF) - 
{'C': 0.1, 'coef0': 0.0, 'degree': 3, 'gamma': 'scale', 'kernel': 'linear'}


⏱️ SVC (TF-IDF) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for SVC (TF-IDF)!

***********************************************************************

 [9/9] Skipped Hyperparameter tuning for XGBoost as there are no hyperparameters for this currently...

***********************************************************************


✅✅✅ All models have been tuned successfully!

#Evaluating all models
tuned_tfidf_df = evaluate_tuned_models(best_models_tfidf, X_train_tfidf, y_train, X_test_tfidf,
                                 y_test, X_val_tfidf, y_val, base_results_tfidf_df)
tuned_tfidf_df

 Evaluating tuned models...

 [1/7] Evaluating best RandomForest model...
 RandomForest model evaluation completed.

 [2/7] Evaluating best DecisionTree model...
 DecisionTree model evaluation completed.

 [3/7] Evaluating best AdaBoost model...
 AdaBoost model evaluation completed.

 [4/7] Evaluating best GradientBoost model...
 GradientBoost model evaluation completed.

 [5/7] Evaluating best LogisticRegression model...
 LogisticRegression model evaluation completed.

 [6/7] Evaluating best KNN model...
 KNN model evaluation completed.

 [7/7] Evaluating best SVM model...
 SVM model evaluation completed.


✅✅✅ All tuned models evaluated successfully!


 Showing the Combined results for both Base and Tuned Models.....

print("\n🔍 Starting Base Model training with Resampled Data (", embedding_name, " Embedding)...\n")

# Calling function to train and evaluate all models
base_results_tuned_tfidf, base_results_tuned_tfidf_df = train_and_evaluate_models(models, X_train_tfidf_resampled, 
                                                                      y_train_resampled, X_test_tfidf_resampled, 
                                                                      y_test_resampled, X_val_tfidf_resampled, 
                                                                      y_val_resampled, embedding_name)
base_results_tuned_tfidf_df

🔍 Starting Base Model training with Resampled Data ( TF-IDF  Embedding)...


🔍 Starting model training and evaluation...


   [1/9] Training RandomForest model...

   ✅ RandomForest training completed in :  0 minute(s) 0 second(s)

   [2/9] Training DecisionTree model...

   ✅ DecisionTree training completed in :  0 minute(s) 0 second(s)

   [3/9] Training NaiveBayes model...

   ✅ NaiveBayes training completed in :  0 minute(s) 0 second(s)

   [4/9] Training AdaBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/ensemble/_weight_boosting.py:527: FutureWarning: The SAMME.R algorithm (the default) is deprecated and will be removed in 1.6. Use the SAMME algorithm to circumvent this warning.
  warnings.warn(

   ✅ AdaBoost training completed in :  0 minute(s) 0 second(s)

   [5/9] Training GradientBoost model...

   ✅ GradientBoost training completed in :  0 minute(s) 4 second(s)

   [6/9] Training LogisticRegression model...

   ✅ LogisticRegression training completed in :  0 minute(s) 0 second(s)

   [7/9] Training KNN model...

   ✅ KNN training completed in :  0 minute(s) 0 second(s)

   [8/9] Training SVM model...

   ✅ SVM training completed in :  0 minute(s) 0 second(s)

   [9/9] Training XGBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/xgboost/core.py:158: UserWarning: [14:30:02] WARNING: /workspace/src/learner.cc:740: 
Parameters: { "use_label_encoder" } are not used.

  warnings.warn(smsg, UserWarning)

   ✅ XGBoost training completed in :  0 minute(s) 0 second(s)

✅✅✅ All models trained and evaluated successfully!!!!!

# Perform Grid Search for all models
best_models_resampled_tfidf = {}

print("\n🔍 Starting hyperparameter tuning for all models with Resampled Data (", embedding_name, " Embedding)...\n")

for i, (name, model) in enumerate(models.items(), start = 1):
    if name in param_grids and param_grids[name]:  # Skip models with no params (e.g., Naive Bayes)
        print(f"\n    [{i}/{len(models)}] Started Hyperparameter tuning for {name}...")
        best_models_resampled_tfidf[name] = perform_grid_search(model, param_grids[name], X_train_tfidf_resampled,
                                                               y_train_resampled, X_test_tfidf_resampled, 
                                                               y_test_resampled, X_val_tfidf_resampled, 
                                                               y_val_resampled, embedding_name)
        print("***********************************************************************")
    else:
        print(f"\n [{i}/{len(models)}] Skipped Hyperparameter tuning for {name} as there are no hyperparameters for this...")
        print("\n***********************************************************************")

print("\n\n✅✅✅ All models have been tuned successfully!\n")

🔍 Starting hyperparameter tuning for all models with Resampled Data ( TF-IDF  Embedding)...


    [1/9] Started Hyperparameter tuning for RandomForest...

🔍 Starting Grid Search for RandomForest (TF-IDF)...

Fitting 5 folds for each of 2160 candidates, totalling 10800 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for RandomForest (TF-IDF) - 
{'criterion': 'entropy', 'max_depth': 25, 'max_features': 'log2', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 100, 'random_state': 42}


⏱️ RandomForest (TF-IDF) Grid Search Time: 2 min(s) 15 sec(s)


 Grid Search completed for RandomForest (TF-IDF)!

***********************************************************************

    [2/9] Started Hyperparameter tuning for DecisionTree...

🔍 Starting Grid Search for DecisionTree (TF-IDF)...

Fitting 5 folds for each of 1800 candidates, totalling 9000 fits

 Best Parameters found for DecisionTree (TF-IDF) - 
{'criterion': 'gini', 'max_depth': 20, 'max_features': 'sqrt', 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'random_state': 42}


⏱️ DecisionTree (TF-IDF) Grid Search Time: 0 min(s) 3 sec(s)


 Grid Search completed for DecisionTree (TF-IDF)!

***********************************************************************

 [3/9] Skipped Hyperparameter tuning for NaiveBayes as there are no hyperparameters for this...

***********************************************************************

    [4/9] Started Hyperparameter tuning for AdaBoost...

🔍 Starting Grid Search for AdaBoost (TF-IDF)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for AdaBoost (TF-IDF) - 
{'algorithm': 'SAMME', 'learning_rate': 1.0, 'n_estimators': 300, 'random_state': 42}


⏱️ AdaBoost (TF-IDF) Grid Search Time: 0 min(s) 5 sec(s)


 Grid Search completed for AdaBoost (TF-IDF)!

***********************************************************************

    [5/9] Started Hyperparameter tuning for GradientBoost...

🔍 Starting Grid Search for GradientBoosting (TF-IDF)...

Fitting 5 folds for each of 972 candidates, totalling 4860 fits

 Best Parameters found for GradientBoosting (TF-IDF) - 
{'learning_rate': 0.2, 'max_depth': 5, 'max_features': 'log2', 'min_samples_leaf': 1, 'min_samples_split': 10, 'n_estimators': 100, 'random_state': 42, 'subsample': 0.8}


⏱️ GradientBoosting (TF-IDF) Grid Search Time: 0 min(s) 37 sec(s)


 Grid Search completed for GradientBoosting (TF-IDF)!

***********************************************************************

    [6/9] Started Hyperparameter tuning for LogisticRegression...

🔍 Starting Grid Search for LogisticRegression (TF-IDF)...

Fitting 5 folds for each of 8 candidates, totalling 40 fits

 Best Parameters found for LogisticRegression (TF-IDF) - 
{'C': 10, 'solver': 'lbfgs'}


⏱️ LogisticRegression (TF-IDF) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for LogisticRegression (TF-IDF)!

***********************************************************************

    [7/9] Started Hyperparameter tuning for KNN...

🔍 Starting Grid Search for KNeighbors (TF-IDF)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for KNeighbors (TF-IDF) - 
{'metric': 'manhattan', 'n_neighbors': 3, 'weights': 'distance'}


⏱️ KNeighbors (TF-IDF) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for KNeighbors (TF-IDF)!

***********************************************************************

    [8/9] Started Hyperparameter tuning for SVM...

🔍 Starting Grid Search for SVC (TF-IDF)...

Fitting 5 folds for each of 162 candidates, totalling 810 fits

 Best Parameters found for SVC (TF-IDF) - 
{'C': 10, 'coef0': 0.0, 'degree': 3, 'gamma': 'scale', 'kernel': 'poly'}


⏱️ SVC (TF-IDF) Grid Search Time: 0 min(s) 5 sec(s)


 Grid Search completed for SVC (TF-IDF)!

***********************************************************************

 [9/9] Skipped Hyperparameter tuning for XGBoost as there are no hyperparameters for this...

***********************************************************************


✅✅✅ All models have been tuned successfully!

#Evaluating all models
tuned_resampled_tfidf_df = evaluate_tuned_models(best_models_resampled_tfidf, X_train_tfidf_resampled, 
                                                      y_train_resampled, X_test_tfidf_resampled, y_test_resampled, 
                                                      X_val_tfidf_resampled, y_val_resampled, base_results_tuned_tfidf_df)
tuned_resampled_tfidf_df

 Evaluating tuned models...

 [1/7] Evaluating best RandomForest model...
 RandomForest model evaluation completed.

 [2/7] Evaluating best DecisionTree model...
 DecisionTree model evaluation completed.

 [3/7] Evaluating best AdaBoost model...
 AdaBoost model evaluation completed.

 [4/7] Evaluating best GradientBoost model...
 GradientBoost model evaluation completed.

 [5/7] Evaluating best LogisticRegression model...
 LogisticRegression model evaluation completed.

 [6/7] Evaluating best KNN model...
 KNN model evaluation completed.

 [7/7] Evaluating best SVM model...
 SVM model evaluation completed.


✅✅✅ All tuned models evaluated successfully!


 Showing the Combined results for both Base and Tuned Models.....

embedding_name = "BoW"

print("\n🔍 Starting Base Model training for all models (", embedding_name, " Embedding)...\n")

# Calling function to train and evaluate all models
base_results_bow, base_results_bow_df = train_and_evaluate_models(models, X_train_bow, y_train, 
                                                          X_test_bow, y_test, X_val_bow, y_val, embedding_name)
base_results_bow_df

🔍 Starting Base Model training for all models ( BoW  Embedding)...


🔍 Starting model training and evaluation...


   [1/9] Training RandomForest model...

   ✅ RandomForest training completed in :  0 minute(s) 0 second(s)

   [2/9] Training DecisionTree model...

   ✅ DecisionTree training completed in :  0 minute(s) 0 second(s)

   [3/9] Training NaiveBayes model...

   ✅ NaiveBayes training completed in :  0 minute(s) 0 second(s)

   [4/9] Training AdaBoost model...

   ✅ AdaBoost training completed in :  0 minute(s) 0 second(s)

   [5/9] Training GradientBoost model...

   ✅ GradientBoost training completed in :  0 minute(s) 0 second(s)

   [6/9] Training LogisticRegression model...

   ✅ LogisticRegression training completed in :  0 minute(s) 0 second(s)

   [7/9] Training KNN model...

   ✅ KNN training completed in :  0 minute(s) 0 second(s)

/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/ensemble/_weight_boosting.py:527: FutureWarning: The SAMME.R algorithm (the default) is deprecated and will be removed in 1.6. Use the SAMME algorithm to circumvent this warning.
  warnings.warn(

   [8/9] Training SVM model...

   ✅ SVM training completed in :  0 minute(s) 0 second(s)

   [9/9] Training XGBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/xgboost/core.py:158: UserWarning: [14:33:11] WARNING: /workspace/src/learner.cc:740: 
Parameters: { "use_label_encoder" } are not used.

  warnings.warn(smsg, UserWarning)

   ✅ XGBoost training completed in :  0 minute(s) 0 second(s)

✅✅✅ All models trained and evaluated successfully!!!!!

# Perform Grid Search for all models
best_models_bow = {}

print("\n🔍 Starting hyperparameter tuning for all models (", embedding_name, " Embedding)...\n")

for i, (name, model) in enumerate(models.items(), start = 1):
    if name in param_grids and param_grids[name]:  # Skip models with no params (e.g., Naive Bayes)
        print(f"\n [{i}/{len(models)}] Started Hyperparameter tuning for {name}...")
        best_models_bow[name] = perform_grid_search(model, param_grids[name], X_train_bow, y_train, 
                                                X_test_bow, y_test, X_val_bow, y_val, embedding_name)
        print("***********************************************************************")
    else:
        print(f"\n [{i}/{len(models)}] Skipped Hyperparameter tuning for {name} as there are no hyperparameters for this currently...")
        print("\n***********************************************************************")

print("\n\n✅✅✅ All models have been tuned successfully!\n")

🔍 Starting hyperparameter tuning for all models ( BoW  Embedding)...


 [1/9] Started Hyperparameter tuning for RandomForest...

🔍 Starting Grid Search for RandomForest (BoW)...

Fitting 5 folds for each of 2160 candidates, totalling 10800 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for RandomForest (BoW) - 
{'criterion': 'gini', 'max_depth': 5, 'max_features': None, 'max_leaf_nodes': None, 'min_samples_leaf': 5, 'min_samples_split': 2, 'n_estimators': 100, 'random_state': 42}


⏱️ RandomForest (BoW) Grid Search Time: 1 min(s) 50 sec(s)


 Grid Search completed for RandomForest (BoW)!

***********************************************************************

 [2/9] Started Hyperparameter tuning for DecisionTree...

🔍 Starting Grid Search for DecisionTree (BoW)...

Fitting 5 folds for each of 1800 candidates, totalling 9000 fits

 Best Parameters found for DecisionTree (BoW) - 
{'criterion': 'gini', 'max_depth': 5, 'max_features': None, 'max_leaf_nodes': None, 'min_samples_leaf': 10, 'min_samples_split': 2, 'random_state': 42}


⏱️ DecisionTree (BoW) Grid Search Time: 0 min(s) 2 sec(s)


 Grid Search completed for DecisionTree (BoW)!

***********************************************************************

 [3/9] Skipped Hyperparameter tuning for NaiveBayes as there are no hyperparameters for this currently...

***********************************************************************

 [4/9] Started Hyperparameter tuning for AdaBoost...

🔍 Starting Grid Search for AdaBoost (BoW)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for AdaBoost (BoW) - 
{'algorithm': 'SAMME', 'learning_rate': 0.1, 'n_estimators': 50, 'random_state': 42}


⏱️ AdaBoost (BoW) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for AdaBoost (BoW)!

***********************************************************************

 [5/9] Started Hyperparameter tuning for GradientBoost...

🔍 Starting Grid Search for GradientBoosting (BoW)...

Fitting 5 folds for each of 972 candidates, totalling 4860 fits

 Best Parameters found for GradientBoosting (BoW) - 
{'learning_rate': 0.01, 'max_depth': 3, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 50, 'random_state': 42, 'subsample': 0.8}


⏱️ GradientBoosting (BoW) Grid Search Time: 0 min(s) 35 sec(s)


 Grid Search completed for GradientBoosting (BoW)!

***********************************************************************

 [6/9] Started Hyperparameter tuning for LogisticRegression...

🔍 Starting Grid Search for LogisticRegression (BoW)...

Fitting 5 folds for each of 8 candidates, totalling 40 fits

 Best Parameters found for LogisticRegression (BoW) - 
{'C': 0.01, 'solver': 'liblinear'}


⏱️ LogisticRegression (BoW) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for LogisticRegression (BoW)!

***********************************************************************

 [7/9] Started Hyperparameter tuning for KNN...

🔍 Starting Grid Search for KNeighbors (BoW)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for KNeighbors (BoW) - 
{'metric': 'euclidean', 'n_neighbors': 9, 'weights': 'uniform'}


⏱️ KNeighbors (BoW) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for KNeighbors (BoW)!

***********************************************************************

 [8/9] Started Hyperparameter tuning for SVM...

🔍 Starting Grid Search for SVC (BoW)...

Fitting 5 folds for each of 162 candidates, totalling 810 fits

 Best Parameters found for SVC (BoW) - 
{'C': 0.1, 'coef0': 0.0, 'degree': 3, 'gamma': 'scale', 'kernel': 'linear'}


⏱️ SVC (BoW) Grid Search Time: 1 min(s) 53 sec(s)


 Grid Search completed for SVC (BoW)!

***********************************************************************

 [9/9] Skipped Hyperparameter tuning for XGBoost as there are no hyperparameters for this currently...

***********************************************************************


✅✅✅ All models have been tuned successfully!

#Evaluating all models
tuned_bow_df = evaluate_tuned_models(best_models_bow, X_train_bow, y_train, X_test_bow,
                                 y_test, X_val_bow, y_val, base_results_bow_df)
tuned_bow_df

 Evaluating tuned models...

 [1/7] Evaluating best RandomForest model...
 RandomForest model evaluation completed.

 [2/7] Evaluating best DecisionTree model...
 DecisionTree model evaluation completed.

 [3/7] Evaluating best AdaBoost model...
 AdaBoost model evaluation completed.

 [4/7] Evaluating best GradientBoost model...
 GradientBoost model evaluation completed.

 [5/7] Evaluating best LogisticRegression model...
 LogisticRegression model evaluation completed.

 [6/7] Evaluating best KNN model...
 KNN model evaluation completed.

 [7/7] Evaluating best SVM model...
 SVM model evaluation completed.


✅✅✅ All tuned models evaluated successfully!


 Showing the Combined results for both Base and Tuned Models.....

print("\n🔍 Starting Base Model training with Resampled Data (", embedding_name, " Embedding)...\n")

# Calling function to train and evaluate all models
base_results_tuned_bow, base_results_tuned_bow_df = train_and_evaluate_models(models, X_train_bow_resampled, 
                                                                      y_train_resampled, X_test_bow_resampled, 
                                                                      y_test_resampled, X_val_bow_resampled, 
                                                                      y_val_resampled, embedding_name)
base_results_tuned_bow_df

🔍 Starting Base Model training with Resampled Data ( BoW  Embedding)...


🔍 Starting model training and evaluation...


   [1/9] Training RandomForest model...

   ✅ RandomForest training completed in :  0 minute(s) 0 second(s)

   [2/9] Training DecisionTree model...

   ✅ DecisionTree training completed in :  0 minute(s) 0 second(s)

   [3/9] Training NaiveBayes model...

   ✅ NaiveBayes training completed in :  0 minute(s) 0 second(s)

   [4/9] Training AdaBoost model...

   ✅ AdaBoost training completed in :  0 minute(s) 0 second(s)

   [5/9] Training GradientBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/ensemble/_weight_boosting.py:527: FutureWarning: The SAMME.R algorithm (the default) is deprecated and will be removed in 1.6. Use the SAMME algorithm to circumvent this warning.
  warnings.warn(

   ✅ GradientBoost training completed in :  0 minute(s) 0 second(s)

   [6/9] Training LogisticRegression model...

   ✅ LogisticRegression training completed in :  0 minute(s) 0 second(s)

   [7/9] Training KNN model...

   ✅ KNN training completed in :  0 minute(s) 0 second(s)

   [8/9] Training SVM model...

   ✅ SVM training completed in :  0 minute(s) 0 second(s)

   [9/9] Training XGBoost model...

/home/opc/miniconda3/lib/python3.12/site-packages/xgboost/core.py:158: UserWarning: [14:37:35] WARNING: /workspace/src/learner.cc:740: 
Parameters: { "use_label_encoder" } are not used.

  warnings.warn(smsg, UserWarning)

   ✅ XGBoost training completed in :  0 minute(s) 0 second(s)

✅✅✅ All models trained and evaluated successfully!!!!!

# Perform Grid Search for all models
best_models_resampled_bow = {}

print("\n🔍 Starting hyperparameter tuning for all models with Resampled Data (", embedding_name, " Embedding)...\n")

for i, (name, model) in enumerate(models.items(), start = 1):
    if name in param_grids and param_grids[name]:  # Skip models with no params (e.g., Naive Bayes)
        print(f"\n    [{i}/{len(models)}] Started Hyperparameter tuning for {name}...")
        best_models_resampled_bow[name] = perform_grid_search(model, param_grids[name], X_train_bow_resampled,
                                                               y_train_resampled, X_test_bow_resampled, 
                                                               y_test_resampled, X_val_bow_resampled, 
                                                               y_val_resampled, embedding_name)
        print("***********************************************************************")
    else:
        print(f"\n [{i}/{len(models)}] Skipped Hyperparameter tuning for {name} as there are no hyperparameters for this...")
        print("\n***********************************************************************")

print("\n\n✅✅✅ All models have been tuned successfully!\n")

🔍 Starting hyperparameter tuning for all models with Resampled Data ( BoW  Embedding)...


    [1/9] Started Hyperparameter tuning for RandomForest...

🔍 Starting Grid Search for RandomForest (BoW)...

Fitting 5 folds for each of 2160 candidates, totalling 10800 fits

/home/opc/miniconda3/lib/python3.12/site-packages/numpy/ma/core.py:2820: RuntimeWarning: invalid value encountered in cast
  _data = np.array(data, dtype=dtype, copy=copy,

 Best Parameters found for RandomForest (BoW) - 
{'criterion': 'gini', 'max_depth': 10, 'max_features': None, 'max_leaf_nodes': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'n_estimators': 50, 'random_state': 42}


⏱️ RandomForest (BoW) Grid Search Time: 1 min(s) 58 sec(s)


 Grid Search completed for RandomForest (BoW)!

***********************************************************************

    [2/9] Started Hyperparameter tuning for DecisionTree...

🔍 Starting Grid Search for DecisionTree (BoW)...

Fitting 5 folds for each of 1800 candidates, totalling 9000 fits

 Best Parameters found for DecisionTree (BoW) - 
{'criterion': 'gini', 'max_depth': 15, 'max_features': None, 'max_leaf_nodes': None, 'min_samples_leaf': 2, 'min_samples_split': 2, 'random_state': 42}


⏱️ DecisionTree (BoW) Grid Search Time: 0 min(s) 1 sec(s)


 Grid Search completed for DecisionTree (BoW)!

***********************************************************************

 [3/9] Skipped Hyperparameter tuning for NaiveBayes as there are no hyperparameters for this...

***********************************************************************

    [4/9] Started Hyperparameter tuning for AdaBoost...

🔍 Starting Grid Search for AdaBoost (BoW)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for AdaBoost (BoW) - 
{'algorithm': 'SAMME', 'learning_rate': 1.0, 'n_estimators': 300, 'random_state': 42}


⏱️ AdaBoost (BoW) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for AdaBoost (BoW)!

***********************************************************************

    [5/9] Started Hyperparameter tuning for GradientBoost...

🔍 Starting Grid Search for GradientBoosting (BoW)...

Fitting 5 folds for each of 972 candidates, totalling 4860 fits

 Best Parameters found for GradientBoosting (BoW) - 
{'learning_rate': 0.1, 'max_depth': 3, 'max_features': 'sqrt', 'min_samples_leaf': 1, 'min_samples_split': 5, 'n_estimators': 100, 'random_state': 42, 'subsample': 0.9}


⏱️ GradientBoosting (BoW) Grid Search Time: 0 min(s) 39 sec(s)


 Grid Search completed for GradientBoosting (BoW)!

***********************************************************************

    [6/9] Started Hyperparameter tuning for LogisticRegression...

🔍 Starting Grid Search for LogisticRegression (BoW)...

Fitting 5 folds for each of 8 candidates, totalling 40 fits

 Best Parameters found for LogisticRegression (BoW) - 
{'C': 0.1, 'solver': 'liblinear'}


⏱️ LogisticRegression (BoW) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for LogisticRegression (BoW)!

***********************************************************************

    [7/9] Started Hyperparameter tuning for KNN...

🔍 Starting Grid Search for KNeighbors (BoW)...

Fitting 5 folds for each of 16 candidates, totalling 80 fits

 Best Parameters found for KNeighbors (BoW) - 
{'metric': 'euclidean', 'n_neighbors': 9, 'weights': 'distance'}


⏱️ KNeighbors (BoW) Grid Search Time: 0 min(s) 0 sec(s)


 Grid Search completed for KNeighbors (BoW)!

***********************************************************************

    [8/9] Started Hyperparameter tuning for SVM...

🔍 Starting Grid Search for SVC (BoW)...

Fitting 5 folds for each of 162 candidates, totalling 810 fits

 Best Parameters found for SVC (BoW) - 
{'C': 10, 'coef0': 0.0, 'degree': 3, 'gamma': 'scale', 'kernel': 'rbf'}


⏱️ SVC (BoW) Grid Search Time: 1 min(s) 7 sec(s)


 Grid Search completed for SVC (BoW)!

***********************************************************************

 [9/9] Skipped Hyperparameter tuning for XGBoost as there are no hyperparameters for this...

***********************************************************************


✅✅✅ All models have been tuned successfully!

#Evaluating all models
tuned_resampled_bow_df = evaluate_tuned_models(best_models_resampled_bow, X_train_bow_resampled, 
                                                      y_train_resampled, X_test_bow_resampled, y_test_resampled, 
                                                      X_val_bow_resampled, y_val_resampled, base_results_tuned_bow_df)
tuned_resampled_bow_df

 Evaluating tuned models...

 [1/7] Evaluating best RandomForest model...
 RandomForest model evaluation completed.

 [2/7] Evaluating best DecisionTree model...
 DecisionTree model evaluation completed.

 [3/7] Evaluating best AdaBoost model...
 AdaBoost model evaluation completed.

 [4/7] Evaluating best GradientBoost model...
 GradientBoost model evaluation completed.

 [5/7] Evaluating best LogisticRegression model...
 LogisticRegression model evaluation completed.

 [6/7] Evaluating best KNN model...
 KNN model evaluation completed.

 [7/7] Evaluating best SVM model...
 SVM model evaluation completed.


✅✅✅ All tuned models evaluated successfully!


 Showing the Combined results for both Base and Tuned Models.....

# Consolidating all models into one dictionary
all_models = {
    "Word2Vec": {
        "Base_Original": base_results_word2vec,  # Base model (original dataset)
        "Tuned_Original": best_models_resampled_word2vec,  # Tuned model (original dataset)
        "Base_Resampled": base_results_tuned_word2vec,  # Base model (resampled dataset)
        "Tuned_Resampled": best_models_resampled_word2vec  # Tuned model (resampled dataset)
    },
    "GloVe": {
        "Base_Original": base_results_glove,
        "Tuned_Original": best_models_resampled_glove,
        "Base_Resampled": base_results_tuned_glove,
        "Tuned_Resampled": best_models_resampled_glove
    },
    "Sentence Transformer": {
        "Base_Original": base_results_sent_trans,
        "Tuned_Original": best_models_resampled_sent_trans,
        "Base_Resampled": base_results_tuned_sent_trans,
        "Tuned_Resampled": best_models_resampled_sent_trans
    },
    "TF-IDF": {
        "Base_Original": base_results_tfidf,
        "Tuned_Original": best_models_resampled_tfidf,
        "Base_Resampled": base_results_tuned_tfidf,
        "Tuned_Resampled": best_models_resampled_tfidf
    },
    "BoW": {
        "Base_Original": base_results_bow,
        "Tuned_Original": best_models_resampled_bow,
        "Base_Resampled": base_results_tuned_bow,
        "Tuned_Resampled": best_models_resampled_bow
    }
}

print(best_models_word2vec.keys())

dict_keys(['RandomForest', 'DecisionTree', 'AdaBoost', 'GradientBoost', 'LogisticRegression', 'KNN', 'SVM'])

print(tuned_word2vec_df.keys())
tuned_word2vec_df

Index(['Model', 'Type', 'Train Acc', 'Train Prec', 'Train Recall', 'Train F1',
       'Val Acc', 'Val Prec', 'Val Recall', 'Val F1', 'Test Acc', 'Test Prec',
       'Test Recall', 'Test F1'],
      dtype='object')

# Evaluating the best tuned Word2Vec model
print("\nPerformance of Best Model on Train Set --->\n\n", model_performance(best_models_word2vec['RandomForest'], X_train_word2vec, y_train))
print("\nPerformance of Best Model on Test Set --->\n\n", model_performance(best_models_word2vec['RandomForest'], X_test_word2vec, y_test))
print("\nPerformance of Best Model on validation Set --->\n\n", model_performance(best_models_word2vec['RandomForest'], X_val_word2vec, y_val))

Performance of Best Model on Train Set --->

   Accuracy  Recall Precision F1-Score
0   0.8040  0.8040    0.8374   0.7468

Performance of Best Model on Test Set --->

   Accuracy  Recall Precision F1-Score
0   0.7381  0.7381    0.8067   0.6269

Performance of Best Model on validation Set --->

   Accuracy  Recall Precision F1-Score
0   0.7381  0.7381    0.8067   0.6269

y_train_best_pred = best_models_word2vec['RandomForest'].predict(X_train_word2vec)
conf_matrix_train = confusion_matrix(y_train, y_train_best_pred)

y_test_best_pred = best_models_word2vec['RandomForest'].predict(X_test_word2vec)
conf_matrix_test = confusion_matrix(y_test, y_test_best_pred)

y_val_best_pred = best_models_word2vec['RandomForest'].predict(X_val_word2vec)
conf_matrix_val = confusion_matrix(y_val, y_val_best_pred)

# Print Classification Reports
print("\nClassification Reports:")
print("\nTrain Classification Report:\n", classification_report(y_train, y_train_best_pred))
print("\nValidation Classification Report:\n", classification_report(y_val, y_val_best_pred))
print("\nTest Classification Report:\n", classification_report(y_test, y_test_best_pred))

Classification Reports:

Train Classification Report:
               precision    recall  f1-score   support

           0       0.00      0.00      0.00        22
           1       0.79      1.00      0.89       185
           2       0.94      0.37      0.53        43

    accuracy                           0.80       250
   macro avg       0.58      0.46      0.47       250
weighted avg       0.75      0.80      0.75       250


Validation Classification Report:
               precision    recall  f1-score   support

           0       0.00      0.00      0.00         8
           1       0.74      1.00      0.85        62
           2       0.00      0.00      0.00        14

    accuracy                           0.74        84
   macro avg       0.25      0.33      0.28        84
weighted avg       0.54      0.74      0.63        84


Test Classification Report:
               precision    recall  f1-score   support

           0       0.00      0.00      0.00         8
           1       0.74      1.00      0.85        62
           2       0.00      0.00      0.00        14

    accuracy                           0.74        84
   macro avg       0.25      0.33      0.28        84
weighted avg       0.54      0.74      0.63        84

/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/metrics/_classification.py:1531: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, f"{metric.capitalize()} is", len(result))
/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/metrics/_classification.py:1531: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, f"{metric.capitalize()} is", len(result))
/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/metrics/_classification.py:1531: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, f"{metric.capitalize()} is", len(result))
/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/metrics/_classification.py:1531: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, f"{metric.capitalize()} is", len(result))
/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/metrics/_classification.py:1531: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, f"{metric.capitalize()} is", len(result))
/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/metrics/_classification.py:1531: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, f"{metric.capitalize()} is", len(result))
/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/metrics/_classification.py:1531: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, f"{metric.capitalize()} is", len(result))
/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/metrics/_classification.py:1531: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, f"{metric.capitalize()} is", len(result))
/home/opc/miniconda3/lib/python3.12/site-packages/sklearn/metrics/_classification.py:1531: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, f"{metric.capitalize()} is", len(result))

# Plot confusion matrix
print("\nConfusion Matrices:")
plot_confusion_matrix(conf_matrix_train, conf_matrix_test, conf_matrix_val)

Confusion Matrices:

Confusion Matrix for Final Training Set:
 [[  0  21   1]
 [  0 185   0]
 [  0  27  16]]
Confusion Matrix for Final Testing Set:
 [[ 0  8  0]
 [ 0 62  0]
 [ 0 14  0]]
Confusion Matrix for Final Validation Set:
 [[ 0  8  0]
 [ 0 62  0]
 [ 0 14  0]]

# ========== Standard Libraries ==========
import time
import warnings
import pickle
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# ========== Text Processing ==========
import string
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer, PorterStemmer

# nltk.download('punkt')
# nltk.download('stopwords')
# nltk.download('wordnet')

# ========== Word Embeddings ==========
from gensim.models import Word2Vec, KeyedVectors
from gensim.scripts.glove2word2vec import glove2word2vec
from sentence_transformers import SentenceTransformer

# ========== TensorFlow & Keras ==========
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import (
    Dense, LSTM, Embedding, SpatialDropout1D, 
    Dropout, BatchNormalization
)
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.metrics import Precision, Recall, AUC
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint
import tensorflow.keras.backend as K

# ========== Sklearn (Data Processing & Evaluation) ==========
from sklearn.model_selection import train_test_split, RandomizedSearchCV
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import (
    confusion_matrix, classification_report, 
    accuracy_score, f1_score, precision_score, recall_score
)

# ========== Transformers (BERT) ==========
from transformers import (
    BertTokenizer, BertForSequenceClassification, Trainer, TrainingArguments, 
    create_optimizer
)

# ========== Imbalanced Data Handling ==========
from imblearn.over_sampling import SMOTE

# ========== Hyperparameter Tuning ==========
import optuna
import keras_tuner as kt

# ========== PyTorch ==========
import torch
from torch.utils.data import DataLoader, Dataset

# ========== Parallel Processing & Display ==========
from joblib import Parallel, delayed
from IPython.display import display

# Suppress warnings
warnings.filterwarnings("ignore")

WARNING:tensorflow:From C:\Users\lakshman_kumar\anaconda3\Lib\site-packages\tf_keras\src\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.

# Load CSV file into a DataFrame
df = pd.read_excel('dataset.xlsx')
# df = pd.read_excel("C:/Users/pri96/OneDrive/Documents/AI and ML PGP/Capstone Project/NLP - 1 (Chatbot)/Data Set - industrial_safety_and_health_database_with_accidents_description.xlsx")

# Check the first few rows of the DataFrame to ensure it's loaded correctly
df.head()

df.shape

(425, 11)

df[df.duplicated()]

df.drop_duplicates(inplace = True)
df.shape

(425, 11)

# drop unwanted columns
df = df[['Description', 'Accident Level']]

# Defining the mapping for 3-class classification
accident_mapping = {
    "I": "Low",       # Minor incidents
    "II": "Medium",   # Noticeable injuries
    "III": "Medium",  # Grouped with Level II
    "IV": "High",     # Severe injuries
    "V": "High"       # Grouped with Level IV
}

# Apply the mapping
df["Accident Category"] = df["Accident Level"].map(accident_mapping)

# Check new class distribution
class_distribution = df["Accident Category"].value_counts()
print("\nNew Class Distribution:\n", class_distribution)
df.head()

New Class Distribution:
 Accident Category
Low       316
Medium     71
High       38
Name: count, dtype: int64

df = df.drop("Accident Level", axis = 1)

df.head()

stop_words = set(stopwords.words('english'))
lemmatizer = WordNetLemmatizer()

# Preprocess function
def preprocess_text(text):
    # Convert to lowercase
    text = text.lower()

    # Tokenize text
    tokens = word_tokenize(text)

    # Remove stopwords and punctuation
    tokens = [word for word in tokens if word not in stop_words and word not in string.punctuation]

    # Lemmatize tokens
    tokens = [lemmatizer.lemmatize(word) for word in tokens]

    return ' '.join(tokens)

# Apply preprocessing to the text column
df['processed_text'] = df['Description'].apply(preprocess_text)

# Check the processed text
print(df['processed_text'].head())

0    removing drill rod jumbo 08 maintenance superv...
1    activation sodium sulphide pump piping uncoupl...
2    sub-station milpo located level +170 collabora...
3    9:45 approximately nv 1880 cx-695 ob7 personne...
4    approximately 11:45 a.m. circumstance mechanic...
Name: processed_text, dtype: object

# Label Encoding
le = LabelEncoder()
df["Accident Category"] = le.fit_transform(df["Accident Category"])

# Display updated DataFrame
df.head()

# Data Splitting between Independent and Target Variables
X = df['processed_text']
y = to_categorical(df['Accident Category'])

# Spliting train, test and validation sets
X_train, X_tmp, y_train, y_tmp = train_test_split(X, y, test_size=0.3, random_state=42)
X_test, X_val, y_test, y_val = train_test_split(X_tmp, y_tmp, test_size=0.5, random_state=42)

# Creating a list of all words in our data
words_list = [item.split(" ") for item in df['processed_text'].values]

# Creating an instance of Word2Vec
vec_size = 300
model_W2V = Word2Vec(words_list, vector_size = vec_size, min_count = 1, window = 5, workers = 6)

# Checking the size of the vocabulary
print("Length of the vocabulary is", len(list(model_W2V.wv.key_to_index)))

Length of the vocabulary is 3257

# Checking the word embedding of a random word
word = "maintenance"
model_W2V.wv[word]

array([ 3.48525681e-03,  8.64117127e-03, -3.63103463e-03,  7.00145122e-03,
        2.62077618e-03, -1.17080742e-02,  3.64604872e-03,  1.92784742e-02,
        1.98191684e-03,  1.44947658e-03, -4.59722476e-03, -8.73077009e-03,
        2.31851591e-04, -4.32454934e-03, -3.73398559e-03, -9.56455339e-03,
        4.36930219e-03,  1.05325063e-03,  4.66557592e-03,  3.50010814e-03,
       -3.22716008e-03, -3.88410548e-03,  8.34404677e-03, -2.44172011e-03,
        9.35433712e-03,  7.79789640e-04, -7.85898976e-03,  2.18380033e-03,
       -5.98587235e-03, -5.55957109e-03,  2.44889059e-03, -7.19311927e-03,
        1.52920221e-03, -1.18533708e-03, -1.01386348e-03,  5.31726936e-03,
        4.52647125e-03, -1.00498060e-02, -2.61241244e-03,  1.23880967e-03,
       -4.22623754e-03,  6.89252221e-04,  1.62721984e-03, -9.29862354e-03,
        4.35516995e-04,  6.71059173e-03,  2.63219629e-03,  5.59536100e-04,
       -3.30209057e-03,  9.29943100e-03,  2.87728244e-03,  9.05995490e-04,
       -2.27275654e-03,  5.05616772e-04, -5.01833492e-05,  1.22812847e-02,
        3.07597406e-03, -2.07180413e-03,  5.73670631e-03, -1.52329085e-04,
       -3.44764069e-03, -2.99923564e-03, -3.38619016e-03,  6.95030810e-03,
        2.53041554e-03, -3.65306769e-05,  2.97747483e-03, -1.73743567e-04,
       -4.70602000e-03, -7.02780671e-04, -6.79157965e-04,  2.02943082e-03,
        9.02485009e-03, -9.72994883e-03,  1.78457005e-03,  1.94212329e-03,
       -6.04991615e-03, -1.15094322e-03, -2.61879642e-03,  6.34800596e-03,
       -1.84285326e-03, -8.37770011e-03,  1.76031579e-04,  1.48549527e-02,
        4.67598112e-03,  3.98907764e-03, -1.15549490e-02, -4.09892621e-03,
        9.38089471e-03,  1.07396871e-03,  1.13399681e-02, -5.99893741e-03,
        3.36478301e-03,  7.92559003e-04,  1.09497681e-02,  1.01296669e-02,
        7.16145150e-03, -1.72427844e-03, -1.03586158e-02,  2.33237376e-03,
        2.64236145e-03,  7.66771380e-04,  5.20970719e-03,  3.15729156e-03,
        3.05289775e-03, -6.24412624e-03, -4.73178411e-03,  2.45879800e-03,
       -7.41687650e-03,  7.92527280e-04, -1.34151867e-02, -8.17360636e-03,
        1.49409391e-03,  7.38824299e-03,  6.23042975e-03, -6.85069535e-04,
        5.71479395e-05,  5.28701872e-04,  1.27114970e-02, -9.16231144e-03,
        1.97186600e-03,  7.14868447e-03,  2.58751330e-03, -2.31683766e-03,
       -2.01479392e-03, -5.14226325e-04,  7.28297420e-03, -1.08441664e-02,
       -2.96398444e-04,  8.37573200e-04, -6.39497330e-06,  1.37658957e-02,
        3.71348200e-04, -8.94925278e-03,  9.71491332e-04,  4.81776707e-03,
       -1.95292686e-03, -9.11988318e-03, -1.05287712e-02, -1.22341411e-02,
        4.08137962e-03, -1.27271069e-02, -8.76591366e-04,  4.15996742e-03,
        5.70741016e-03, -5.74736157e-03, -9.20566730e-03, -3.37608019e-03,
        5.57725690e-03, -1.54094445e-03, -2.01967359e-03, -1.12333475e-02,
       -5.02824178e-03, -5.81459748e-03,  1.53823220e-03,  2.20609596e-03,
       -7.77246198e-03, -6.87148795e-03, -3.08151753e-03,  5.44569548e-03,
        2.17053387e-03,  8.38086382e-03, -1.42652616e-02,  9.31006297e-03,
       -8.53411946e-03,  1.20311708e-03,  2.68310867e-03, -1.34901900e-04,
        5.46024181e-03,  1.42333899e-02, -1.83150161e-03,  1.26165431e-03,
        1.59119011e-03,  9.15272045e-04,  2.45288917e-04,  1.39702973e-03,
        2.05665798e-04, -7.58598931e-03, -2.36653979e-03, -3.57684703e-03,
       -2.76715937e-03,  6.85872789e-03, -7.98979495e-03, -3.75350704e-03,
       -2.50185933e-03,  2.44102906e-03,  9.12124291e-03,  8.59382655e-03,
        2.02867598e-03, -5.01752319e-03,  5.65353082e-04, -2.23194459e-03,
       -1.02087576e-02,  2.91801407e-03,  1.41853723e-03, -9.51897446e-03,
        9.64243547e-04, -8.72460287e-03,  5.32122608e-03, -1.19631877e-03,
       -1.06262872e-02,  1.79392204e-03, -2.99173803e-03, -7.42590288e-03,
       -3.09608132e-03, -6.41594362e-03, -1.82939169e-03,  4.82577970e-03,
       -2.16075638e-03, -3.56769841e-03,  2.74779566e-04, -6.17780443e-03,
       -5.41708618e-03, -3.53910145e-03,  3.71374818e-03, -9.69311595e-03,
       -4.97647049e-03, -1.78121757e-02, -7.16648670e-03, -9.38273035e-03,
        4.61380230e-03, -4.83069220e-04, -8.60675890e-03, -5.52197406e-03,
       -3.27027799e-03, -3.92544316e-03, -1.19583472e-03, -4.83049266e-03,
       -8.70392472e-03,  3.85474670e-03,  8.39740224e-03,  5.14202635e-04,
       -2.29622424e-03,  6.30549388e-03, -3.78606492e-03,  2.06082687e-03,
       -1.70487387e-04,  1.24348677e-03,  2.30706716e-03, -1.39479591e-02,
       -1.74332852e-03, -4.60212165e-03, -2.61660526e-03, -2.41694809e-03,
        6.83025923e-04, -8.24054424e-03, -3.71580478e-03,  5.99947385e-03,
       -3.24113597e-03,  1.09460140e-02,  4.51592728e-03,  7.88399921e-05,
        2.27576098e-03,  7.45723781e-04, -1.13826981e-02, -6.69197179e-03,
        9.76948999e-03,  7.62299821e-03, -1.17588397e-02, -2.94017093e-03,
        8.29335302e-03,  2.50557717e-03, -6.60427206e-04, -1.39435111e-02,
       -1.02236802e-02, -2.36781850e-03,  6.89992215e-03,  4.91064088e-03,
       -1.91721821e-03,  1.79544138e-03, -7.65953958e-03, -2.22460390e-03,
       -2.90843449e-03, -5.23652742e-03,  1.03422524e-02,  2.42306478e-03,
        9.10631567e-03,  5.58621157e-03, -3.98197025e-03, -1.95320393e-03,
        4.43369569e-03,  1.43775949e-03, -5.31423418e-03,  5.75942267e-03,
       -5.59499196e-04, -1.39621770e-04, -8.09023809e-03,  5.52912196e-03,
        1.81095453e-03,  9.83814523e-03, -2.29452271e-03,  9.62027162e-03,
        8.36732239e-03,  4.40407963e-03,  6.48060860e-03,  1.58980973e-02,
        2.69319722e-03, -7.49491900e-03,  5.20230085e-03, -4.10765409e-03],
      dtype=float32)

# Retrieving the words present in the Word2Vec model's vocabulary
words = list(model_W2V.wv.key_to_index.keys())

# Retrieving word vectors for all the words present in the model's vocabulary
wvs = model_W2V.wv[words].tolist()

# Creating a dictionary of words and their corresponding vectors
word_vector_dict = dict(zip(words, wvs))

def average_vectorizer_Word2Vec(doc):
    # Initializing a feature vector for the sentence
    feature_vector = np.zeros((vec_size,), dtype="float64")

    # Creating a list of words in the sentence that are present in the model vocabulary
    words_in_vocab = [word for word in doc.split() if word in words]

    # adding the vector representations of the words
    for word in words_in_vocab:
        feature_vector += np.array(word_vector_dict[word])

    # Dividing by the number of words to get the average vector
    if len(words_in_vocab) != 0:
        feature_vector /= len(words_in_vocab)

    return feature_vector

# creating a dataframe of the vectorized documents - splitting train, test and validation sets for word2vec

X_train_wv = pd.DataFrame(X_train.apply(average_vectorizer_Word2Vec).tolist(), columns=['Feature '+str(i) for i in range(vec_size)])
X_val_wv = pd.DataFrame(X_val.apply(average_vectorizer_Word2Vec).tolist(), columns=['Feature '+str(i) for i in range(vec_size)])
X_test_wv = pd.DataFrame(X_test.apply(average_vectorizer_Word2Vec).tolist(), columns=['Feature '+str(i) for i in range(vec_size)])

print(X_train_wv.shape, X_val_wv.shape, X_test_wv.shape)

(297, 300) (64, 300) (64, 300)

# load the Stanford GloVe model
filename = 'glove.6B.100d.txt.word2vec'
# filename = "C:/Users/pri96/OneDrive/Documents/AI and ML PGP/Capstone Project/NLP - 1 (Chatbot)/glove.6B.100d.txt.word2vec"

glove_model = KeyedVectors.load_word2vec_format(filename, binary=False)

# Checking the size of the vocabulary
print("Length of the vocabulary is", len(glove_model.index_to_key))

Length of the vocabulary is 400000

# Checking the word embedding of a random word
word = "maintenance"
glove_model[word]

array([-6.0166e-01, -3.7872e-02, -3.4488e-01, -5.5135e-02, -1.9668e-01,
       -9.2608e-01, -3.7591e-01,  8.3588e-01, -3.9453e-02,  9.0746e-01,
        6.4115e-01, -2.0984e-01,  8.2506e-01,  1.4270e-02, -1.4302e-01,
       -7.6316e-01,  5.8217e-01,  7.0070e-02,  4.9675e-01,  5.6788e-01,
       -3.9395e-01, -1.5133e-01, -1.0797e-01, -3.6763e-01, -4.7262e-04,
       -3.8080e-01, -9.4109e-01, -1.2140e-01, -3.9608e-01, -4.3378e-02,
       -8.1921e-01, -5.9489e-02,  6.0219e-02, -9.4380e-02, -4.7681e-01,
        8.7606e-01, -4.3544e-02, -2.3628e-01,  8.5798e-01,  1.0755e-01,
        6.9542e-01, -2.6819e-01,  3.7445e-01, -3.1330e-01,  3.3271e-01,
        3.0675e-01, -2.2009e-01, -2.8861e-01, -4.3272e-02, -6.1320e-01,
        4.8036e-01, -1.6773e-01, -7.7702e-01,  6.4762e-01,  3.0346e-01,
       -7.9420e-01,  6.1859e-01, -8.8403e-01,  2.3665e+00,  4.1936e-01,
       -5.3211e-02, -3.5368e-01,  3.8188e-01,  5.7688e-01,  3.2484e-01,
        2.2086e-02, -1.2007e-01, -7.2210e-01,  7.0829e-01,  2.1601e-01,
       -6.0639e-01, -7.1401e-02, -8.9727e-02,  5.7614e-02,  9.3982e-01,
       -4.6520e-01,  3.8118e-01,  5.4458e-01, -6.4240e-01,  2.2124e-01,
        2.4511e-01,  1.5908e-01,  1.9719e-01,  5.5232e-01, -1.8323e+00,
        6.9847e-01,  3.8601e-01, -3.2733e-01, -9.0524e-01,  3.2656e-01,
       -2.2065e-01, -6.7076e-01,  1.7316e-01,  3.5935e-01, -1.7889e-01,
        1.9655e-01,  8.2041e-01, -2.4492e-01,  8.1646e-01, -1.5404e-01],
      dtype=float32)

# Retrieving the words present in the GloVe model's vocabulary
glove_words = glove_model.index_to_key

# Creating a dictionary of words and their corresponding vectors
glove_word_vector_dict = dict(zip(glove_model.index_to_key,list(glove_model.vectors)))

vec_size=100

def average_vectorizer_GloVe(doc):
    # Initializing a feature vector for the sentence
    feature_vector = np.zeros((vec_size,), dtype="float64")

    # Creating a list of words in the sentence that are present in the model vocabulary
    words_in_vocab = [word for word in doc.split() if word in glove_words]

    # adding the vector representations of the words
    for word in words_in_vocab:
        feature_vector += np.array(glove_word_vector_dict[word])

    # Dividing by the number of words to get the average vector
    if len(words_in_vocab) != 0:
        feature_vector /= len(words_in_vocab)

    return feature_vector

# creating a dataframe of the vectorized documents - splitting train, test and validation sets for glove

X_train_gl = pd.DataFrame(X_train.apply(average_vectorizer_GloVe).tolist(), columns=['Feature '+str(i) for i in range(vec_size)])
X_val_gl = pd.DataFrame(X_val.apply(average_vectorizer_GloVe).tolist(), columns=['Feature '+str(i) for i in range(vec_size)])
X_test_gl = pd.DataFrame(X_test.apply(average_vectorizer_GloVe).tolist(), columns=['Feature '+str(i) for i in range(vec_size)])

print(X_train_gl.shape, X_val_gl.shape, X_test_gl.shape)

(297, 100) (64, 100) (64, 100)

# Load pre-trained model
model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')

# Encode sentences to get sentence embeddings - splitting train, test and validation sets for sentence transformer

X_train_st = model.encode(X_train.values)
X_val_st = model.encode(X_val.values)
X_test_st = model.encode(X_test.values)

# Check the shape of the embeddings
print(X_train_st.shape, X_val_st.shape, X_test_st.shape)

(297, 384) (64, 384) (64, 384)

def plot_confusion_matrices_nn(model, 
                            train_predictors, train_target, 
                            val_predictors, val_target, 
                            test_predictors, test_target):
    """
    Plots confusion matrices for a neural network model on training, validation, and test datasets
    
    Parameters:
        model : The trained neural network model used for predictions
        model : The trained neural network model used for predictions
        train_predictors, val_predictors, test_predictors : Independent variables
        train_target, val_target, test_target : Target set
    
    """
    # Create a figure to hold three subplots
    fig, axes = plt.subplots(1, 3, figsize=(18, 6))
    
    # Define the labels for the confusion matrix
    label_list = ['High', 'Low', 'Medium']

    # List of datasets and titles for each subplot
    datasets = [(train_predictors, train_target), 
                (val_predictors, val_target), 
                (test_predictors, test_target)]
    titles = ['Training Set', 'Validation Set', 'Test Set']
    
    for i, (predictors, target) in enumerate(datasets):
        pred = np.argmax(model.predict(predictors), axis=1)  # Make predictions using the classifier.
        cm = confusion_matrix(target, pred)  # Compute the confusion matrix.
        
        # Plot the confusion matrix on the respective subplot
        sns.heatmap(cm, annot=True, fmt='.0f', cmap='Blues', 
                    xticklabels=label_list, yticklabels=label_list,
                    ax=axes[i])
        
        axes[i].set_ylabel('Actual')  # Label for the y-axis.
        axes[i].set_xlabel('Predicted')  # Label for the x-axis.
        axes[i].set_title(f'Confusion Matrix - {titles[i]}')  # Title for each subplot

    # Adjust layout and show the plot
    plt.tight_layout()
    plt.show()

def model_performance_classification_sklearn_nn(model, 
                                              train_predictors, train_target, 
                                              val_predictors, val_target, 
                                              test_predictors, test_target):
    """
    Computes classification performance metrics for a neural network model 
    on training, validation, and test datasets.
    
    Parameters:
        model : The trained neural network model used for predictions
        train_predictors, val_predictors, test_predictors : Independent variables
        train_target, val_target, test_target : Target set
    
    Returns:
        A DataFrame containing Accuracy, Recall, Precision, and F1-score 
        for training, validation, and test datasets
    """
    # Predictions for train, validation, and test datasets
    train_pred = np.argmax(model.predict(train_predictors), axis=1)
    val_pred = np.argmax(model.predict(val_predictors), axis=1)
    test_pred = np.argmax(model.predict(test_predictors), axis=1)

    # Compute metrics for each dataset
    train_acc = accuracy_score(train_target, train_pred)
    val_acc = accuracy_score(val_target, val_pred)
    test_acc = accuracy_score(test_target, test_pred)

    train_recall = recall_score(train_target, train_pred, average='weighted')
    val_recall = recall_score(val_target, val_pred, average='weighted')
    test_recall = recall_score(test_target, test_pred, average='weighted')

    train_precision = precision_score(train_target, train_pred, average='weighted')
    val_precision = precision_score(val_target, val_pred, average='weighted')
    test_precision = precision_score(test_target, test_pred, average='weighted')

    train_f1 = f1_score(train_target, train_pred, average='weighted')
    val_f1 = f1_score(val_target, val_pred, average='weighted')
    test_f1 = f1_score(test_target, test_pred, average='weighted')

    # Create a DataFrame to store the computed metrics
    df_perf = pd.DataFrame({
        "Train": [train_acc, train_recall, train_precision, train_f1],
        "Validation": [val_acc, val_recall, val_precision, val_f1],
        "Test": [test_acc, test_recall, test_precision, test_f1],
    }, index=["Accuracy", "Recall", "Precision", "F1_score"])

    return df_perf  # Return the transposed DataFrame with the metrics

# Function to create a base neural network model
def create_base_model(input_shape, output_shape):
    """
    Creates and compiles a base neural network model with fully connected layers, batch normalization, and dropout for regularization
    
    Parameters:
        input_shape : Number of input features for the model
        output_shape : The number of output classes for classification
    
    Returns: A compiled Sequential model ready for training
    """
    model = Sequential([
        Dense(32, input_dim=input_shape, activation='relu'),
        BatchNormalization(),
        Dropout(0.6),
        Dense(16, activation='relu'),
        BatchNormalization(),
        Dropout(0.4),
        Dense(output_shape, activation='softmax')
    ])
    model.compile(optimizer=Adam(), loss='categorical_crossentropy', metrics=['accuracy'])
    return model

# Function to build the tuned model with Keras Tuner
def build_tuned_model(hp, input_shape, output_shape):
    """
    Builds and compiles a neural network model with tunable hyperparameters using Keras Tuner for optimization
    
    Parameters:
        hp : Hyperparameter tuning object from Keras Tuner
        input_shape : The number of input features for the model
        output_shape : The number of output classes for classification
    
    Returns: A compiled Sequential model with tunable layers and learning rate
    """
    model = Sequential([
        Dense(hp.Choice('num_neurons_1', [32, 64, 128]), activation='relu', input_dim=input_shape),
        BatchNormalization(),
        Dropout(0.6),
        Dense(hp.Choice('num_neurons_2', [16, 32, 64]), activation='relu'),
        BatchNormalization(),
        Dropout(0.4),
        Dense(output_shape, activation='softmax')
    ])

    optimizer = Adam(learning_rate=hp.Choice('learning_rate', [0.001, 0.01, 0.1]))
    model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])
    
    return model

# Function to train and evaluate with Keras Tuner
def train_and_evaluate_model(X_train, X_test, y_train, y_test, use_tuned_model=False):
    """
    Trains and evaluates a neural network model using either a base architecture or a hyperparameter-tuned model
    
    Parameters:
        X_train, X_test : Independent variables
        y_train, y_test : Target variables set
        use_tuned_model : Boolean flag to determine whether to use a tuner or not
    
    Returns:
        best_model : The trained neural network model
        precision : Precision score of the model
        recall : Recall score of the model
        f1 : F1-score of the model
    """
    input_shape = X_train.shape[1]
    output_shape = y_train.shape[1]

    print("X_train shape:", X_train.shape)
    print("y_train shape:", y_train.shape)

    # Callbacks: Early Stopping, ReduceLROnPlateau, and ModelCheckpoint
    # callbacks = [
    #     EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True, verbose=1),
    #     ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, verbose=1),
    #     ModelCheckpoint('best_model.keras', monitor='val_accuracy', save_best_only=True, verbose=1)
    # ]

    if use_tuned_model:
        tuner = kt.RandomSearch(
            lambda hp: build_tuned_model(hp, input_shape, output_shape),  
            objective='val_accuracy',
            max_trials=10,
            executions_per_trial=1,
            directory='tuner_results',
            project_name='nn_tuning'
        )

        tuner.search(X_train, y_train, epochs=10, validation_split=0.2, verbose=2) #, callbacks=callbacks)

        # Get the best hyperparameters
        best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]

        # Build and print the summary of the best model
        best_model = tuner.hypermodel.build(best_hps)
        print("\n📌 Best Tuned Model Summary:")
        best_model.summary()  # Prints the model architecture
    else:
        best_model = create_base_model(input_shape, output_shape)
        print("\n📌 Base Model Summary:")
        best_model.summary()  # Prints base model summary

    best_model.fit(X_train, y_train, epochs=55, batch_size=32, validation_data=(X_test, y_test), verbose=2)#, callbacks=callbacks)

    # Evaluate the model
    y_pred = best_model.predict(X_test)
    y_pred = np.argmax(y_pred, axis=1)
    y_test_indices = np.argmax(y_test, axis=1)

    precision = precision_score(y_test_indices, y_pred, average='weighted')
    recall = recall_score(y_test_indices, y_pred, average='weighted')
    f1 = f1_score(y_test_indices, y_pred, average='weighted')

    print(f"Precision: {precision:.4f}")
    print(f"Recall: {recall:.4f}")
    print(f"F1-Score: {f1:.4f}")

    return best_model, precision, recall, f1

print("Evaluating base model:")
base_model_wv, base_precision_wv, base_recall_wv, base_f1score_wv = train_and_evaluate_model(X_train_wv, X_val_wv, y_train, y_val, use_tuned_model=False)

Evaluating base model:
X_train shape: (297, 300)
y_train shape: (297, 3)

📌 Base Model Summary:

Model: "sequential"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Layer (type)                         ┃ Output Shape                ┃         Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ dense (Dense)                        │ (None, 32)                  │           9,632 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization                  │ (None, 32)                  │             128 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout (Dropout)                    │ (None, 32)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_1 (Dense)                      │ (None, 16)                  │             528 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_1                │ (None, 16)                  │              64 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_1 (Dropout)                  │ (None, 16)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_2 (Dense)                      │ (None, 3)                   │              51 │
└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘

 Total params: 10,403 (40.64 KB)

 Trainable params: 10,307 (40.26 KB)

 Non-trainable params: 96 (384.00 B)

Epoch 1/55
10/10 - 2s - 159ms/step - accuracy: 0.3771 - loss: 1.2034 - val_accuracy: 0.7812 - val_loss: 1.0741
Epoch 2/55
10/10 - 0s - 11ms/step - accuracy: 0.3569 - loss: 1.1883 - val_accuracy: 0.7812 - val_loss: 1.0501
Epoch 3/55
10/10 - 0s - 9ms/step - accuracy: 0.4242 - loss: 1.1107 - val_accuracy: 0.7812 - val_loss: 1.0281
Epoch 4/55
10/10 - 0s - 8ms/step - accuracy: 0.4276 - loss: 1.0920 - val_accuracy: 0.7812 - val_loss: 1.0073
Epoch 5/55
10/10 - 0s - 8ms/step - accuracy: 0.4983 - loss: 1.0183 - val_accuracy: 0.7812 - val_loss: 0.9862
Epoch 6/55
10/10 - 0s - 9ms/step - accuracy: 0.5859 - loss: 1.0014 - val_accuracy: 0.7812 - val_loss: 0.9656
Epoch 7/55
10/10 - 0s - 9ms/step - accuracy: 0.5993 - loss: 0.9791 - val_accuracy: 0.7812 - val_loss: 0.9450
Epoch 8/55
10/10 - 0s - 8ms/step - accuracy: 0.6734 - loss: 0.9699 - val_accuracy: 0.7812 - val_loss: 0.9260
Epoch 9/55
10/10 - 0s - 8ms/step - accuracy: 0.6970 - loss: 0.9483 - val_accuracy: 0.7812 - val_loss: 0.9077
Epoch 10/55
10/10 - 0s - 9ms/step - accuracy: 0.7037 - loss: 0.9353 - val_accuracy: 0.7812 - val_loss: 0.8897
Epoch 11/55
10/10 - 0s - 9ms/step - accuracy: 0.7104 - loss: 0.9175 - val_accuracy: 0.7812 - val_loss: 0.8713
Epoch 12/55
10/10 - 0s - 9ms/step - accuracy: 0.7172 - loss: 0.9085 - val_accuracy: 0.7812 - val_loss: 0.8539
Epoch 13/55
10/10 - 0s - 8ms/step - accuracy: 0.7037 - loss: 0.8953 - val_accuracy: 0.7812 - val_loss: 0.8389
Epoch 14/55
10/10 - 0s - 8ms/step - accuracy: 0.7138 - loss: 0.8655 - val_accuracy: 0.7812 - val_loss: 0.8227
Epoch 15/55
10/10 - 0s - 9ms/step - accuracy: 0.7172 - loss: 0.8577 - val_accuracy: 0.7812 - val_loss: 0.8072
Epoch 16/55
10/10 - 0s - 9ms/step - accuracy: 0.7205 - loss: 0.8473 - val_accuracy: 0.7812 - val_loss: 0.7938
Epoch 17/55
10/10 - 0s - 8ms/step - accuracy: 0.7172 - loss: 0.8392 - val_accuracy: 0.7812 - val_loss: 0.7800
Epoch 18/55
10/10 - 0s - 8ms/step - accuracy: 0.7239 - loss: 0.8199 - val_accuracy: 0.7812 - val_loss: 0.7671
Epoch 19/55
10/10 - 0s - 8ms/step - accuracy: 0.7172 - loss: 0.8235 - val_accuracy: 0.7812 - val_loss: 0.7556
Epoch 20/55
10/10 - 0s - 9ms/step - accuracy: 0.7273 - loss: 0.8052 - val_accuracy: 0.7812 - val_loss: 0.7451
Epoch 21/55
10/10 - 0s - 9ms/step - accuracy: 0.7239 - loss: 0.8014 - val_accuracy: 0.7812 - val_loss: 0.7369
Epoch 22/55
10/10 - 0s - 8ms/step - accuracy: 0.7239 - loss: 0.7972 - val_accuracy: 0.7812 - val_loss: 0.7283
Epoch 23/55
10/10 - 0s - 8ms/step - accuracy: 0.7205 - loss: 0.8166 - val_accuracy: 0.7812 - val_loss: 0.7202
Epoch 24/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7917 - val_accuracy: 0.7812 - val_loss: 0.7142
Epoch 25/55
10/10 - 0s - 8ms/step - accuracy: 0.7306 - loss: 0.7714 - val_accuracy: 0.7812 - val_loss: 0.7086
Epoch 26/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7715 - val_accuracy: 0.7812 - val_loss: 0.7044
Epoch 27/55
10/10 - 0s - 9ms/step - accuracy: 0.7239 - loss: 0.7856 - val_accuracy: 0.7812 - val_loss: 0.7007
Epoch 28/55
10/10 - 0s - 9ms/step - accuracy: 0.7273 - loss: 0.7687 - val_accuracy: 0.7812 - val_loss: 0.6979
Epoch 29/55
10/10 - 0s - 7ms/step - accuracy: 0.7273 - loss: 0.7800 - val_accuracy: 0.7812 - val_loss: 0.6961
Epoch 30/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7751 - val_accuracy: 0.7812 - val_loss: 0.6948
Epoch 31/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7662 - val_accuracy: 0.7812 - val_loss: 0.6924
Epoch 32/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7758 - val_accuracy: 0.7812 - val_loss: 0.6898
Epoch 33/55
10/10 - 0s - 9ms/step - accuracy: 0.7273 - loss: 0.7785 - val_accuracy: 0.7812 - val_loss: 0.6873
Epoch 34/55
10/10 - 0s - 9ms/step - accuracy: 0.7239 - loss: 0.7852 - val_accuracy: 0.7812 - val_loss: 0.6853
Epoch 35/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7681 - val_accuracy: 0.7812 - val_loss: 0.6839
Epoch 36/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7640 - val_accuracy: 0.7812 - val_loss: 0.6818
Epoch 37/55
10/10 - 0s - 9ms/step - accuracy: 0.7273 - loss: 0.7703 - val_accuracy: 0.7812 - val_loss: 0.6808
Epoch 38/55
10/10 - 0s - 9ms/step - accuracy: 0.7273 - loss: 0.7672 - val_accuracy: 0.7812 - val_loss: 0.6782
Epoch 39/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7603 - val_accuracy: 0.7812 - val_loss: 0.6759
Epoch 40/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7618 - val_accuracy: 0.7812 - val_loss: 0.6759
Epoch 41/55
10/10 - 0s - 7ms/step - accuracy: 0.7273 - loss: 0.7670 - val_accuracy: 0.7812 - val_loss: 0.6764
Epoch 42/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7819 - val_accuracy: 0.7812 - val_loss: 0.6767
Epoch 43/55
10/10 - 0s - 9ms/step - accuracy: 0.7273 - loss: 0.7508 - val_accuracy: 0.7812 - val_loss: 0.6754
Epoch 44/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7366 - val_accuracy: 0.7812 - val_loss: 0.6737
Epoch 45/55
10/10 - 0s - 7ms/step - accuracy: 0.7239 - loss: 0.7541 - val_accuracy: 0.7812 - val_loss: 0.6716
Epoch 46/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7687 - val_accuracy: 0.7812 - val_loss: 0.6687
Epoch 47/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7539 - val_accuracy: 0.7812 - val_loss: 0.6801
Epoch 48/55
10/10 - 0s - 9ms/step - accuracy: 0.7273 - loss: 0.7672 - val_accuracy: 0.7812 - val_loss: 0.6814
Epoch 49/55
10/10 - 0s - 9ms/step - accuracy: 0.7273 - loss: 0.7362 - val_accuracy: 0.7812 - val_loss: 0.6829
Epoch 50/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7392 - val_accuracy: 0.7812 - val_loss: 0.6800
Epoch 51/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7311 - val_accuracy: 0.7812 - val_loss: 0.6758
Epoch 52/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7465 - val_accuracy: 0.7812 - val_loss: 0.6801
Epoch 53/55
10/10 - 0s - 9ms/step - accuracy: 0.7273 - loss: 0.7337 - val_accuracy: 0.7812 - val_loss: 0.6798
Epoch 54/55
10/10 - 0s - 9ms/step - accuracy: 0.7273 - loss: 0.7182 - val_accuracy: 0.7812 - val_loss: 0.6657
Epoch 55/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7146 - val_accuracy: 0.7812 - val_loss: 0.6694
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 15ms/step
Precision: 0.6104
Recall: 0.7812
F1-Score: 0.6853

plot_confusion_matrices_nn(base_model_wv, 
                        X_train_wv, np.argmax(y_train, axis=1), 
                        X_val_wv, np.argmax(y_val, axis=1), 
                        X_test_wv, np.argmax(y_test, axis=1))

10/10 ━━━━━━━━━━━━━━━━━━━━ 0s 8ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 14ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 18ms/step

#Calculating different metrics on base_wv data
base_metrics_wv = model_performance_classification_sklearn_nn(base_model_wv, 
                        X_train_wv, np.argmax(y_train, axis=1), 
                        X_val_wv, np.argmax(y_val, axis=1), 
                        X_test_wv, np.argmax(y_test, axis=1))
print("Model performance:\n")
base_metrics_wv

10/10 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step 
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 12ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 13ms/step
Model performance:

print("\nEvaluating tuned model:")
tuned_model_wv, tuned_precision_wv, tuned_recall_wv, tuned_f1score_wv = train_and_evaluate_model(X_train_wv, X_val_wv, y_train, y_val, use_tuned_model=True)

Evaluating tuned model:
X_train shape: (297, 300)
y_train shape: (297, 3)
Reloading Tuner from tuner_results\nn_tuning\tuner0.json

📌 Best Tuned Model Summary:

Model: "sequential_1"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Layer (type)                         ┃ Output Shape                ┃         Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ dense_3 (Dense)                      │ (None, 64)                  │          19,264 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_2                │ (None, 64)                  │             256 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_2 (Dropout)                  │ (None, 64)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_4 (Dense)                      │ (None, 64)                  │           4,160 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_3                │ (None, 64)                  │             256 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_3 (Dropout)                  │ (None, 64)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_5 (Dense)                      │ (None, 3)                   │             195 │
└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘

 Total params: 24,131 (94.26 KB)

 Trainable params: 23,875 (93.26 KB)

 Non-trainable params: 256 (1.00 KB)

Epoch 1/55
10/10 - 2s - 156ms/step - accuracy: 0.5219 - loss: 1.5471 - val_accuracy: 0.7812 - val_loss: 1.0427
Epoch 2/55
10/10 - 0s - 8ms/step - accuracy: 0.6027 - loss: 1.4388 - val_accuracy: 0.7812 - val_loss: 1.5339
Epoch 3/55
10/10 - 0s - 8ms/step - accuracy: 0.6195 - loss: 1.0367 - val_accuracy: 0.7812 - val_loss: 0.7662
Epoch 4/55
10/10 - 0s - 8ms/step - accuracy: 0.6431 - loss: 0.9491 - val_accuracy: 0.7812 - val_loss: 0.8297
Epoch 5/55
10/10 - 0s - 8ms/step - accuracy: 0.6734 - loss: 0.8464 - val_accuracy: 0.7812 - val_loss: 0.7566
Epoch 6/55
10/10 - 0s - 8ms/step - accuracy: 0.7239 - loss: 0.8647 - val_accuracy: 0.7812 - val_loss: 0.6708
Epoch 7/55
10/10 - 0s - 8ms/step - accuracy: 0.6566 - loss: 0.8727 - val_accuracy: 0.7812 - val_loss: 0.6630
Epoch 8/55
10/10 - 0s - 8ms/step - accuracy: 0.7037 - loss: 0.8684 - val_accuracy: 0.7812 - val_loss: 0.6683
Epoch 9/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.8297 - val_accuracy: 0.7812 - val_loss: 0.6716
Epoch 10/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7844 - val_accuracy: 0.7812 - val_loss: 0.6755
Epoch 11/55
10/10 - 0s - 8ms/step - accuracy: 0.7138 - loss: 0.8042 - val_accuracy: 0.7812 - val_loss: 0.6804
Epoch 12/55
10/10 - 0s - 8ms/step - accuracy: 0.7172 - loss: 0.8056 - val_accuracy: 0.7812 - val_loss: 0.7035
Epoch 13/55
10/10 - 0s - 8ms/step - accuracy: 0.7104 - loss: 0.8069 - val_accuracy: 0.7812 - val_loss: 0.7060
Epoch 14/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.8200 - val_accuracy: 0.7812 - val_loss: 0.6822
Epoch 15/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7843 - val_accuracy: 0.7812 - val_loss: 0.6653
Epoch 16/55
10/10 - 0s - 8ms/step - accuracy: 0.7138 - loss: 0.7919 - val_accuracy: 0.7812 - val_loss: 0.6863
Epoch 17/55
10/10 - 0s - 8ms/step - accuracy: 0.7037 - loss: 0.8498 - val_accuracy: 0.7812 - val_loss: 0.6795
Epoch 18/55
10/10 - 0s - 8ms/step - accuracy: 0.7239 - loss: 0.8225 - val_accuracy: 0.7812 - val_loss: 0.6791
Epoch 19/55
10/10 - 0s - 8ms/step - accuracy: 0.7239 - loss: 0.7925 - val_accuracy: 0.7812 - val_loss: 0.7611
Epoch 20/55
10/10 - 0s - 8ms/step - accuracy: 0.7138 - loss: 0.8366 - val_accuracy: 0.7812 - val_loss: 0.7241
Epoch 21/55
10/10 - 0s - 9ms/step - accuracy: 0.7037 - loss: 0.8984 - val_accuracy: 0.7812 - val_loss: 0.7231
Epoch 22/55
10/10 - 0s - 9ms/step - accuracy: 0.6768 - loss: 0.8671 - val_accuracy: 0.7812 - val_loss: 0.6659
Epoch 23/55
10/10 - 0s - 8ms/step - accuracy: 0.7071 - loss: 0.8388 - val_accuracy: 0.7812 - val_loss: 0.6921
Epoch 24/55
10/10 - 0s - 8ms/step - accuracy: 0.7172 - loss: 0.8320 - val_accuracy: 0.7812 - val_loss: 0.7004
Epoch 25/55
10/10 - 0s - 8ms/step - accuracy: 0.7138 - loss: 0.7911 - val_accuracy: 0.7812 - val_loss: 0.6919
Epoch 26/55
10/10 - 0s - 8ms/step - accuracy: 0.7172 - loss: 0.8316 - val_accuracy: 0.7812 - val_loss: 0.6509
Epoch 27/55
10/10 - 0s - 8ms/step - accuracy: 0.7138 - loss: 0.8051 - val_accuracy: 0.7812 - val_loss: 0.6626
Epoch 28/55
10/10 - 0s - 8ms/step - accuracy: 0.7239 - loss: 0.8161 - val_accuracy: 0.7812 - val_loss: 0.8015
Epoch 29/55
10/10 - 0s - 8ms/step - accuracy: 0.7104 - loss: 0.8107 - val_accuracy: 0.7812 - val_loss: 0.6507
Epoch 30/55
10/10 - 0s - 8ms/step - accuracy: 0.7003 - loss: 0.8246 - val_accuracy: 0.7812 - val_loss: 0.6726
Epoch 31/55
10/10 - 0s - 8ms/step - accuracy: 0.6902 - loss: 0.8809 - val_accuracy: 0.7812 - val_loss: 0.6593
Epoch 32/55
10/10 - 0s - 8ms/step - accuracy: 0.7205 - loss: 0.8521 - val_accuracy: 0.7812 - val_loss: 0.7113
Epoch 33/55
10/10 - 0s - 8ms/step - accuracy: 0.7138 - loss: 0.7889 - val_accuracy: 0.7812 - val_loss: 0.6537
Epoch 34/55
10/10 - 0s - 8ms/step - accuracy: 0.7172 - loss: 0.8741 - val_accuracy: 0.7812 - val_loss: 0.7288
Epoch 35/55
10/10 - 0s - 8ms/step - accuracy: 0.7104 - loss: 0.8160 - val_accuracy: 0.7812 - val_loss: 0.6296
Epoch 36/55
10/10 - 0s - 8ms/step - accuracy: 0.7104 - loss: 0.7947 - val_accuracy: 0.7812 - val_loss: 0.6791
Epoch 37/55
10/10 - 0s - 8ms/step - accuracy: 0.7104 - loss: 0.8718 - val_accuracy: 0.7344 - val_loss: 0.6769
Epoch 38/55
10/10 - 0s - 8ms/step - accuracy: 0.6970 - loss: 0.8527 - val_accuracy: 0.7812 - val_loss: 0.7320
Epoch 39/55
10/10 - 0s - 8ms/step - accuracy: 0.7138 - loss: 0.8422 - val_accuracy: 0.7812 - val_loss: 0.6515
Epoch 40/55
10/10 - 0s - 8ms/step - accuracy: 0.7003 - loss: 0.8733 - val_accuracy: 0.7812 - val_loss: 0.6937
Epoch 41/55
10/10 - 0s - 8ms/step - accuracy: 0.7239 - loss: 0.8250 - val_accuracy: 0.7812 - val_loss: 0.6894
Epoch 42/55
10/10 - 0s - 8ms/step - accuracy: 0.7239 - loss: 0.8254 - val_accuracy: 0.7812 - val_loss: 0.6786
Epoch 43/55
10/10 - 0s - 8ms/step - accuracy: 0.7071 - loss: 0.7946 - val_accuracy: 0.7812 - val_loss: 0.6402
Epoch 44/55
10/10 - 0s - 8ms/step - accuracy: 0.7172 - loss: 0.8043 - val_accuracy: 0.7812 - val_loss: 0.6466
Epoch 45/55
10/10 - 0s - 8ms/step - accuracy: 0.7172 - loss: 0.8261 - val_accuracy: 0.7812 - val_loss: 0.7345
Epoch 46/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.8329 - val_accuracy: 0.7812 - val_loss: 0.6716
Epoch 47/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.8204 - val_accuracy: 0.7812 - val_loss: 0.6811
Epoch 48/55
10/10 - 0s - 8ms/step - accuracy: 0.6835 - loss: 0.8045 - val_accuracy: 0.7812 - val_loss: 0.6714
Epoch 49/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.8099 - val_accuracy: 0.7812 - val_loss: 0.6682
Epoch 50/55
10/10 - 0s - 8ms/step - accuracy: 0.7138 - loss: 0.8206 - val_accuracy: 0.7812 - val_loss: 0.6773
Epoch 51/55
10/10 - 0s - 9ms/step - accuracy: 0.7003 - loss: 0.8362 - val_accuracy: 0.7812 - val_loss: 0.6582
Epoch 52/55
10/10 - 0s - 9ms/step - accuracy: 0.6835 - loss: 0.8120 - val_accuracy: 0.7812 - val_loss: 0.6703
Epoch 53/55
10/10 - 0s - 9ms/step - accuracy: 0.6801 - loss: 0.8425 - val_accuracy: 0.7812 - val_loss: 0.6768
Epoch 54/55
10/10 - 0s - 8ms/step - accuracy: 0.7205 - loss: 0.8826 - val_accuracy: 0.7812 - val_loss: 0.7280
Epoch 55/55
10/10 - 0s - 8ms/step - accuracy: 0.7138 - loss: 0.7928 - val_accuracy: 0.7812 - val_loss: 0.7673
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 15ms/step
Precision: 0.6104
Recall: 0.7812
F1-Score: 0.6853

plot_confusion_matrices_nn(tuned_model_wv, 
                        X_train_wv, np.argmax(y_train, axis=1), 
                        X_val_wv, np.argmax(y_val, axis=1), 
                        X_test_wv, np.argmax(y_test, axis=1))

10/10 ━━━━━━━━━━━━━━━━━━━━ 0s 8ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 13ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 15ms/step

#Calculating different metrics on tuned_wv data
tuned_metrics_wv = model_performance_classification_sklearn_nn(tuned_model_wv, 
                        X_train_wv, np.argmax(y_train, axis=1), 
                        X_val_wv, np.argmax(y_val, axis=1), 
                        X_test_wv, np.argmax(y_test, axis=1))
print("Model performance:\n")
tuned_metrics_wv

10/10 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step 
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 15ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 12ms/step
Model performance:

print("Evaluating base model:")
base_model_gl, base_precision_gl, base_recall_gl, base_f1score_gl = train_and_evaluate_model(X_train_gl, X_val_gl, y_train, y_val, use_tuned_model=False)

Evaluating base model:
X_train shape: (297, 100)
y_train shape: (297, 3)

📌 Base Model Summary:

Model: "sequential_2"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Layer (type)                         ┃ Output Shape                ┃         Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ dense_6 (Dense)                      │ (None, 32)                  │           3,232 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_4                │ (None, 32)                  │             128 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_4 (Dropout)                  │ (None, 32)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_7 (Dense)                      │ (None, 16)                  │             528 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_5                │ (None, 16)                  │              64 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_5 (Dropout)                  │ (None, 16)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_8 (Dense)                      │ (None, 3)                   │              51 │
└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘

 Total params: 4,003 (15.64 KB)

 Trainable params: 3,907 (15.26 KB)

 Non-trainable params: 96 (384.00 B)

Epoch 1/55
10/10 - 2s - 193ms/step - accuracy: 0.2997 - loss: 1.4917 - val_accuracy: 0.7812 - val_loss: 1.0463
Epoch 2/55
10/10 - 0s - 8ms/step - accuracy: 0.3636 - loss: 1.4438 - val_accuracy: 0.7656 - val_loss: 1.0334
Epoch 3/55
10/10 - 0s - 8ms/step - accuracy: 0.3636 - loss: 1.3606 - val_accuracy: 0.7812 - val_loss: 1.0166
Epoch 4/55
10/10 - 0s - 8ms/step - accuracy: 0.4108 - loss: 1.1818 - val_accuracy: 0.7812 - val_loss: 1.0032
Epoch 5/55
10/10 - 0s - 8ms/step - accuracy: 0.4040 - loss: 1.2373 - val_accuracy: 0.7812 - val_loss: 0.9874
Epoch 6/55
10/10 - 0s - 9ms/step - accuracy: 0.4478 - loss: 1.1542 - val_accuracy: 0.7812 - val_loss: 0.9733
Epoch 7/55
10/10 - 0s - 8ms/step - accuracy: 0.4343 - loss: 1.1555 - val_accuracy: 0.7812 - val_loss: 0.9610
Epoch 8/55
10/10 - 0s - 8ms/step - accuracy: 0.4848 - loss: 1.0925 - val_accuracy: 0.7812 - val_loss: 0.9517
Epoch 9/55
10/10 - 0s - 9ms/step - accuracy: 0.4680 - loss: 1.0888 - val_accuracy: 0.7812 - val_loss: 0.9399
Epoch 10/55
10/10 - 0s - 8ms/step - accuracy: 0.4781 - loss: 1.1059 - val_accuracy: 0.7812 - val_loss: 0.9266
Epoch 11/55
10/10 - 0s - 8ms/step - accuracy: 0.4848 - loss: 1.0866 - val_accuracy: 0.7812 - val_loss: 0.9157
Epoch 12/55
10/10 - 0s - 8ms/step - accuracy: 0.4949 - loss: 1.0352 - val_accuracy: 0.7812 - val_loss: 0.9052
Epoch 13/55
10/10 - 0s - 8ms/step - accuracy: 0.5152 - loss: 0.9751 - val_accuracy: 0.7812 - val_loss: 0.8937
Epoch 14/55
10/10 - 0s - 8ms/step - accuracy: 0.5219 - loss: 0.9740 - val_accuracy: 0.7812 - val_loss: 0.8825
Epoch 15/55
10/10 - 0s - 8ms/step - accuracy: 0.5286 - loss: 1.0059 - val_accuracy: 0.7812 - val_loss: 0.8725
Epoch 16/55
10/10 - 0s - 9ms/step - accuracy: 0.5354 - loss: 0.9880 - val_accuracy: 0.7812 - val_loss: 0.8577
Epoch 17/55
10/10 - 0s - 9ms/step - accuracy: 0.5286 - loss: 0.9485 - val_accuracy: 0.7812 - val_loss: 0.8381
Epoch 18/55
10/10 - 0s - 8ms/step - accuracy: 0.5892 - loss: 0.9071 - val_accuracy: 0.7812 - val_loss: 0.8208
Epoch 19/55
10/10 - 0s - 8ms/step - accuracy: 0.5791 - loss: 0.8925 - val_accuracy: 0.7812 - val_loss: 0.8102
Epoch 20/55
10/10 - 0s - 8ms/step - accuracy: 0.6364 - loss: 0.8847 - val_accuracy: 0.7812 - val_loss: 0.7993
Epoch 21/55
10/10 - 0s - 9ms/step - accuracy: 0.5825 - loss: 0.8913 - val_accuracy: 0.7812 - val_loss: 0.7958
Epoch 22/55
10/10 - 0s - 9ms/step - accuracy: 0.6094 - loss: 0.8741 - val_accuracy: 0.7812 - val_loss: 0.7892
Epoch 23/55
10/10 - 0s - 8ms/step - accuracy: 0.5791 - loss: 0.9469 - val_accuracy: 0.7812 - val_loss: 0.7831
Epoch 24/55
10/10 - 0s - 9ms/step - accuracy: 0.6027 - loss: 0.8518 - val_accuracy: 0.7812 - val_loss: 0.7792
Epoch 25/55
10/10 - 0s - 9ms/step - accuracy: 0.6599 - loss: 0.8042 - val_accuracy: 0.7812 - val_loss: 0.7668
Epoch 26/55
10/10 - 0s - 9ms/step - accuracy: 0.6364 - loss: 0.8404 - val_accuracy: 0.7812 - val_loss: 0.7561
Epoch 27/55
10/10 - 0s - 9ms/step - accuracy: 0.6599 - loss: 0.8441 - val_accuracy: 0.7812 - val_loss: 0.7461
Epoch 28/55
10/10 - 0s - 9ms/step - accuracy: 0.6936 - loss: 0.7761 - val_accuracy: 0.7812 - val_loss: 0.7392
Epoch 29/55
10/10 - 0s - 8ms/step - accuracy: 0.6633 - loss: 0.8147 - val_accuracy: 0.7812 - val_loss: 0.7323
Epoch 30/55
10/10 - 0s - 10ms/step - accuracy: 0.6566 - loss: 0.8506 - val_accuracy: 0.7812 - val_loss: 0.7249
Epoch 31/55
10/10 - 0s - 9ms/step - accuracy: 0.7003 - loss: 0.7652 - val_accuracy: 0.7812 - val_loss: 0.7200
Epoch 32/55
10/10 - 0s - 8ms/step - accuracy: 0.6902 - loss: 0.7973 - val_accuracy: 0.7812 - val_loss: 0.7151
Epoch 33/55
10/10 - 0s - 8ms/step - accuracy: 0.6599 - loss: 0.8301 - val_accuracy: 0.7812 - val_loss: 0.7136
Epoch 34/55
10/10 - 0s - 8ms/step - accuracy: 0.6869 - loss: 0.7763 - val_accuracy: 0.7812 - val_loss: 0.7091
Epoch 35/55
10/10 - 0s - 8ms/step - accuracy: 0.6700 - loss: 0.8447 - val_accuracy: 0.7812 - val_loss: 0.7085
Epoch 36/55
10/10 - 0s - 9ms/step - accuracy: 0.6970 - loss: 0.7985 - val_accuracy: 0.7812 - val_loss: 0.7066
Epoch 37/55
10/10 - 0s - 8ms/step - accuracy: 0.7071 - loss: 0.7666 - val_accuracy: 0.7812 - val_loss: 0.6955
Epoch 38/55
10/10 - 0s - 8ms/step - accuracy: 0.6835 - loss: 0.7762 - val_accuracy: 0.7812 - val_loss: 0.6899
Epoch 39/55
10/10 - 0s - 9ms/step - accuracy: 0.6768 - loss: 0.8062 - val_accuracy: 0.7812 - val_loss: 0.6881
Epoch 40/55
10/10 - 0s - 8ms/step - accuracy: 0.6970 - loss: 0.8053 - val_accuracy: 0.7812 - val_loss: 0.6887
Epoch 41/55
10/10 - 0s - 8ms/step - accuracy: 0.6835 - loss: 0.8251 - val_accuracy: 0.7812 - val_loss: 0.6859
Epoch 42/55
10/10 - 0s - 8ms/step - accuracy: 0.6970 - loss: 0.7581 - val_accuracy: 0.7812 - val_loss: 0.6686
Epoch 43/55
10/10 - 0s - 8ms/step - accuracy: 0.6801 - loss: 0.7789 - val_accuracy: 0.7812 - val_loss: 0.6548
Epoch 44/55
10/10 - 0s - 8ms/step - accuracy: 0.6835 - loss: 0.7580 - val_accuracy: 0.7812 - val_loss: 0.6452
Epoch 45/55
10/10 - 0s - 8ms/step - accuracy: 0.6902 - loss: 0.7732 - val_accuracy: 0.7812 - val_loss: 0.6417
Epoch 46/55
10/10 - 0s - 8ms/step - accuracy: 0.7172 - loss: 0.7472 - val_accuracy: 0.7812 - val_loss: 0.6399
Epoch 47/55
10/10 - 0s - 8ms/step - accuracy: 0.7003 - loss: 0.7322 - val_accuracy: 0.7812 - val_loss: 0.6360
Epoch 48/55
10/10 - 0s - 8ms/step - accuracy: 0.7441 - loss: 0.6879 - val_accuracy: 0.7812 - val_loss: 0.6326
Epoch 49/55
10/10 - 0s - 8ms/step - accuracy: 0.7407 - loss: 0.7152 - val_accuracy: 0.7812 - val_loss: 0.6259
Epoch 50/55
10/10 - 0s - 8ms/step - accuracy: 0.7172 - loss: 0.7489 - val_accuracy: 0.7812 - val_loss: 0.6215
Epoch 51/55
10/10 - 0s - 9ms/step - accuracy: 0.7306 - loss: 0.7017 - val_accuracy: 0.7812 - val_loss: 0.6194
Epoch 52/55
10/10 - 0s - 9ms/step - accuracy: 0.7475 - loss: 0.7009 - val_accuracy: 0.7656 - val_loss: 0.6188
Epoch 53/55
10/10 - 0s - 9ms/step - accuracy: 0.7374 - loss: 0.6836 - val_accuracy: 0.7656 - val_loss: 0.6154
Epoch 54/55
10/10 - 0s - 8ms/step - accuracy: 0.7374 - loss: 0.7150 - val_accuracy: 0.7656 - val_loss: 0.6138
Epoch 55/55
10/10 - 0s - 8ms/step - accuracy: 0.7104 - loss: 0.6922 - val_accuracy: 0.7656 - val_loss: 0.6113
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 14ms/step
Precision: 0.6174
Recall: 0.7656
F1-Score: 0.6836

plot_confusion_matrices_nn(base_model_gl, 
                        X_train_gl, np.argmax(y_train, axis=1), 
                        X_val_gl, np.argmax(y_val, axis=1), 
                        X_test_gl, np.argmax(y_test, axis=1))

10/10 ━━━━━━━━━━━━━━━━━━━━ 0s 8ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 14ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 17ms/step

#Calculating different metrics on base_gl data
base_metrics_gl = model_performance_classification_sklearn_nn(base_model_gl, 
                        X_train_gl, np.argmax(y_train, axis=1), 
                        X_val_gl, np.argmax(y_val, axis=1), 
                        X_test_gl, np.argmax(y_test, axis=1))
print("Model performance:\n")
base_metrics_gl

10/10 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step 
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 15ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 15ms/step
Model performance:

print("\nEvaluating tuned model:")
tuned_model_gl, tuned_precision_gl, tuned_recall_gl, tuned_f1score_gl = train_and_evaluate_model(X_train_gl, X_val_gl, y_train, y_val, use_tuned_model=True)

Evaluating tuned model:
X_train shape: (297, 100)
y_train shape: (297, 3)
Reloading Tuner from tuner_results\nn_tuning\tuner0.json

📌 Best Tuned Model Summary:

Model: "sequential_3"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Layer (type)                         ┃ Output Shape                ┃         Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ dense_9 (Dense)                      │ (None, 64)                  │           6,464 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_6                │ (None, 64)                  │             256 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_6 (Dropout)                  │ (None, 64)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_10 (Dense)                     │ (None, 64)                  │           4,160 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_7                │ (None, 64)                  │             256 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_7 (Dropout)                  │ (None, 64)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_11 (Dense)                     │ (None, 3)                   │             195 │
└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘

 Total params: 11,331 (44.26 KB)

 Trainable params: 11,075 (43.26 KB)

 Non-trainable params: 256 (1.00 KB)

Epoch 1/55
10/10 - 2s - 150ms/step - accuracy: 0.5185 - loss: 1.5977 - val_accuracy: 0.2031 - val_loss: 2.7048
Epoch 2/55
10/10 - 0s - 8ms/step - accuracy: 0.5859 - loss: 1.4237 - val_accuracy: 0.7500 - val_loss: 1.2646
Epoch 3/55
10/10 - 0s - 8ms/step - accuracy: 0.6431 - loss: 1.0523 - val_accuracy: 0.1406 - val_loss: 1.7134
Epoch 4/55
10/10 - 0s - 8ms/step - accuracy: 0.7138 - loss: 0.8503 - val_accuracy: 0.1406 - val_loss: 3.4333
Epoch 5/55
10/10 - 0s - 8ms/step - accuracy: 0.7003 - loss: 0.8476 - val_accuracy: 0.2031 - val_loss: 1.8284
Epoch 6/55
10/10 - 0s - 8ms/step - accuracy: 0.6667 - loss: 0.8479 - val_accuracy: 0.3125 - val_loss: 1.2455
Epoch 7/55
10/10 - 0s - 8ms/step - accuracy: 0.6768 - loss: 0.8899 - val_accuracy: 0.1875 - val_loss: 2.0798
Epoch 8/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.8084 - val_accuracy: 0.5625 - val_loss: 0.8058
Epoch 9/55
10/10 - 0s - 8ms/step - accuracy: 0.7037 - loss: 0.7673 - val_accuracy: 0.7344 - val_loss: 0.6902
Epoch 10/55
10/10 - 0s - 8ms/step - accuracy: 0.6936 - loss: 0.8170 - val_accuracy: 0.7812 - val_loss: 0.7866
Epoch 11/55
10/10 - 0s - 8ms/step - accuracy: 0.7205 - loss: 0.7991 - val_accuracy: 0.7812 - val_loss: 0.6350
Epoch 12/55
10/10 - 0s - 7ms/step - accuracy: 0.7205 - loss: 0.7827 - val_accuracy: 0.7500 - val_loss: 0.6689
Epoch 13/55
10/10 - 0s - 8ms/step - accuracy: 0.7003 - loss: 0.8058 - val_accuracy: 0.7812 - val_loss: 0.6474
Epoch 14/55
10/10 - 0s - 8ms/step - accuracy: 0.7138 - loss: 0.7577 - val_accuracy: 0.7656 - val_loss: 0.6119
Epoch 15/55
10/10 - 0s - 8ms/step - accuracy: 0.7407 - loss: 0.7603 - val_accuracy: 0.7656 - val_loss: 0.6546
Epoch 16/55
10/10 - 0s - 9ms/step - accuracy: 0.7071 - loss: 0.7889 - val_accuracy: 0.7812 - val_loss: 0.5946
Epoch 17/55
10/10 - 0s - 9ms/step - accuracy: 0.7071 - loss: 0.8293 - val_accuracy: 0.7812 - val_loss: 0.6261
Epoch 18/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7686 - val_accuracy: 0.7031 - val_loss: 0.6400
Epoch 19/55
10/10 - 0s - 8ms/step - accuracy: 0.7172 - loss: 0.7370 - val_accuracy: 0.7656 - val_loss: 0.6287
Epoch 20/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7191 - val_accuracy: 0.7812 - val_loss: 0.5987
Epoch 21/55
10/10 - 0s - 8ms/step - accuracy: 0.7306 - loss: 0.7188 - val_accuracy: 0.7969 - val_loss: 0.5870
Epoch 22/55
10/10 - 0s - 8ms/step - accuracy: 0.7205 - loss: 0.7526 - val_accuracy: 0.7812 - val_loss: 0.5967
Epoch 23/55
10/10 - 0s - 7ms/step - accuracy: 0.7037 - loss: 0.7226 - val_accuracy: 0.7812 - val_loss: 0.5788
Epoch 24/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7003 - val_accuracy: 0.8125 - val_loss: 0.6273
Epoch 25/55
10/10 - 0s - 8ms/step - accuracy: 0.7037 - loss: 0.7922 - val_accuracy: 0.7969 - val_loss: 0.6149
Epoch 26/55
10/10 - 0s - 9ms/step - accuracy: 0.7205 - loss: 0.7717 - val_accuracy: 0.7656 - val_loss: 0.6646
Epoch 27/55
10/10 - 0s - 9ms/step - accuracy: 0.6801 - loss: 0.7756 - val_accuracy: 0.7969 - val_loss: 0.7171
Epoch 28/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7039 - val_accuracy: 0.7812 - val_loss: 0.6819
Epoch 29/55
10/10 - 0s - 8ms/step - accuracy: 0.7003 - loss: 0.8255 - val_accuracy: 0.7969 - val_loss: 0.6819
Epoch 30/55
10/10 - 0s - 8ms/step - accuracy: 0.7340 - loss: 0.6910 - val_accuracy: 0.3906 - val_loss: 1.1639
Epoch 31/55
10/10 - 0s - 9ms/step - accuracy: 0.7104 - loss: 0.7775 - val_accuracy: 0.4375 - val_loss: 0.8605
Epoch 32/55
10/10 - 0s - 9ms/step - accuracy: 0.6599 - loss: 0.8086 - val_accuracy: 0.5469 - val_loss: 0.8356
Epoch 33/55
10/10 - 0s - 8ms/step - accuracy: 0.7172 - loss: 0.7692 - val_accuracy: 0.5781 - val_loss: 0.7901
Epoch 34/55
10/10 - 0s - 8ms/step - accuracy: 0.7374 - loss: 0.7856 - val_accuracy: 0.7344 - val_loss: 0.6985
Epoch 35/55
10/10 - 0s - 8ms/step - accuracy: 0.7138 - loss: 0.7883 - val_accuracy: 0.5938 - val_loss: 0.7900
Epoch 36/55
10/10 - 0s - 8ms/step - accuracy: 0.6902 - loss: 0.8408 - val_accuracy: 0.7656 - val_loss: 0.7307
Epoch 37/55
10/10 - 0s - 8ms/step - accuracy: 0.7138 - loss: 0.7588 - val_accuracy: 0.7812 - val_loss: 0.6859
Epoch 38/55
10/10 - 0s - 8ms/step - accuracy: 0.7104 - loss: 0.7434 - val_accuracy: 0.7812 - val_loss: 0.6886
Epoch 39/55
10/10 - 0s - 8ms/step - accuracy: 0.7205 - loss: 0.6846 - val_accuracy: 0.7500 - val_loss: 0.7500
Epoch 40/55
10/10 - 0s - 8ms/step - accuracy: 0.7205 - loss: 0.7324 - val_accuracy: 0.7812 - val_loss: 0.6904
Epoch 41/55
10/10 - 0s - 8ms/step - accuracy: 0.6869 - loss: 0.7243 - val_accuracy: 0.7812 - val_loss: 0.7042
Epoch 42/55
10/10 - 0s - 9ms/step - accuracy: 0.7003 - loss: 0.7274 - val_accuracy: 0.7656 - val_loss: 0.6796
Epoch 43/55
10/10 - 0s - 8ms/step - accuracy: 0.6936 - loss: 0.7545 - val_accuracy: 0.6406 - val_loss: 0.8439
Epoch 44/55
10/10 - 0s - 8ms/step - accuracy: 0.7071 - loss: 0.7049 - val_accuracy: 0.5625 - val_loss: 0.8626
Epoch 45/55
10/10 - 0s - 8ms/step - accuracy: 0.7441 - loss: 0.6794 - val_accuracy: 0.6406 - val_loss: 0.7700
Epoch 46/55
10/10 - 0s - 8ms/step - accuracy: 0.7239 - loss: 0.7422 - val_accuracy: 0.7500 - val_loss: 0.7466
Epoch 47/55
10/10 - 0s - 9ms/step - accuracy: 0.6902 - loss: 0.8152 - val_accuracy: 0.5469 - val_loss: 1.1836
Epoch 48/55
10/10 - 0s - 9ms/step - accuracy: 0.7138 - loss: 0.7138 - val_accuracy: 0.7969 - val_loss: 0.7167
Epoch 49/55
10/10 - 0s - 9ms/step - accuracy: 0.7003 - loss: 0.7635 - val_accuracy: 0.7812 - val_loss: 0.9787
Epoch 50/55
10/10 - 0s - 8ms/step - accuracy: 0.7003 - loss: 0.8043 - val_accuracy: 0.7812 - val_loss: 0.8504
Epoch 51/55
10/10 - 0s - 8ms/step - accuracy: 0.7205 - loss: 0.7252 - val_accuracy: 0.7969 - val_loss: 0.7702
Epoch 52/55
10/10 - 0s - 8ms/step - accuracy: 0.7508 - loss: 0.6953 - val_accuracy: 0.5625 - val_loss: 0.8852
Epoch 53/55
10/10 - 0s - 8ms/step - accuracy: 0.7104 - loss: 0.8464 - val_accuracy: 0.4844 - val_loss: 1.3372
Epoch 54/55
10/10 - 0s - 8ms/step - accuracy: 0.7273 - loss: 0.7397 - val_accuracy: 0.7031 - val_loss: 0.7212
Epoch 55/55
10/10 - 0s - 8ms/step - accuracy: 0.7407 - loss: 0.7801 - val_accuracy: 0.7969 - val_loss: 0.6393
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 16ms/step
Precision: 0.7213
Recall: 0.7969
F1-Score: 0.7366

plot_confusion_matrices_nn(tuned_model_gl, 
                        X_train_gl, np.argmax(y_train, axis=1), 
                        X_val_gl, np.argmax(y_val, axis=1), 
                        X_test_gl, np.argmax(y_test, axis=1))

10/10 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 14ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 14ms/step

#Calculating different metrics on tuned_gl data
tuned_metrics_gl = model_performance_classification_sklearn_nn(tuned_model_gl, 
                        X_train_gl, np.argmax(y_train, axis=1), 
                        X_val_gl, np.argmax(y_val, axis=1), 
                        X_test_gl, np.argmax(y_test, axis=1))
print("Model performance:\n")
tuned_metrics_gl

10/10 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step 
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 13ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 12ms/step
Model performance:

print("Evaluating base model:")
base_model_st, base_precision_st, base_recall_st, base_f1score_st = train_and_evaluate_model(X_train_st, X_val_st, y_train, y_val, use_tuned_model=False)

Evaluating base model:
X_train shape: (297, 384)
y_train shape: (297, 3)

📌 Base Model Summary:

Model: "sequential_4"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Layer (type)                         ┃ Output Shape                ┃         Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ dense_12 (Dense)                     │ (None, 32)                  │          12,320 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_8                │ (None, 32)                  │             128 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_8 (Dropout)                  │ (None, 32)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_13 (Dense)                     │ (None, 16)                  │             528 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_9                │ (None, 16)                  │              64 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_9 (Dropout)                  │ (None, 16)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_14 (Dense)                     │ (None, 3)                   │              51 │
└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘

 Total params: 13,091 (51.14 KB)

 Trainable params: 12,995 (50.76 KB)

 Non-trainable params: 96 (384.00 B)

Epoch 1/55
10/10 - 1s - 147ms/step - accuracy: 0.3636 - loss: 1.7119 - val_accuracy: 0.5000 - val_loss: 1.0740
Epoch 2/55
10/10 - 0s - 8ms/step - accuracy: 0.3805 - loss: 1.6622 - val_accuracy: 0.6719 - val_loss: 1.0380
Epoch 3/55
10/10 - 0s - 9ms/step - accuracy: 0.3771 - loss: 1.5658 - val_accuracy: 0.7812 - val_loss: 1.0104
Epoch 4/55
10/10 - 0s - 8ms/step - accuracy: 0.3939 - loss: 1.4289 - val_accuracy: 0.7812 - val_loss: 0.9858
Epoch 5/55
10/10 - 0s - 8ms/step - accuracy: 0.4478 - loss: 1.2763 - val_accuracy: 0.7812 - val_loss: 0.9677
Epoch 6/55
10/10 - 0s - 8ms/step - accuracy: 0.4108 - loss: 1.3003 - val_accuracy: 0.7812 - val_loss: 0.9480
Epoch 7/55
10/10 - 0s - 8ms/step - accuracy: 0.4478 - loss: 1.2523 - val_accuracy: 0.7812 - val_loss: 0.9274
Epoch 8/55
10/10 - 0s - 9ms/step - accuracy: 0.4882 - loss: 1.1772 - val_accuracy: 0.7812 - val_loss: 0.9182
Epoch 9/55
10/10 - 0s - 9ms/step - accuracy: 0.5320 - loss: 1.0887 - val_accuracy: 0.7812 - val_loss: 0.9050
Epoch 10/55
10/10 - 0s - 9ms/step - accuracy: 0.4848 - loss: 1.1100 - val_accuracy: 0.7812 - val_loss: 0.8880
Epoch 11/55
10/10 - 0s - 8ms/step - accuracy: 0.4747 - loss: 1.1645 - val_accuracy: 0.7812 - val_loss: 0.8708
Epoch 12/55
10/10 - 0s - 9ms/step - accuracy: 0.5219 - loss: 1.0395 - val_accuracy: 0.7812 - val_loss: 0.8573
Epoch 13/55
10/10 - 0s - 9ms/step - accuracy: 0.5522 - loss: 0.9957 - val_accuracy: 0.7812 - val_loss: 0.8432
Epoch 14/55
10/10 - 0s - 8ms/step - accuracy: 0.6027 - loss: 0.8981 - val_accuracy: 0.7812 - val_loss: 0.8271
Epoch 15/55
10/10 - 0s - 8ms/step - accuracy: 0.6061 - loss: 0.9730 - val_accuracy: 0.7812 - val_loss: 0.8104
Epoch 16/55
10/10 - 0s - 8ms/step - accuracy: 0.6061 - loss: 0.9455 - val_accuracy: 0.7812 - val_loss: 0.7969
Epoch 17/55
10/10 - 0s - 9ms/step - accuracy: 0.6128 - loss: 0.8579 - val_accuracy: 0.7812 - val_loss: 0.7859
Epoch 18/55
10/10 - 0s - 8ms/step - accuracy: 0.6633 - loss: 0.8437 - val_accuracy: 0.7812 - val_loss: 0.7742
Epoch 19/55
10/10 - 0s - 8ms/step - accuracy: 0.6768 - loss: 0.8176 - val_accuracy: 0.7812 - val_loss: 0.7664
Epoch 20/55
10/10 - 0s - 8ms/step - accuracy: 0.6431 - loss: 0.8221 - val_accuracy: 0.7812 - val_loss: 0.7570
Epoch 21/55
10/10 - 0s - 8ms/step - accuracy: 0.6970 - loss: 0.7355 - val_accuracy: 0.7812 - val_loss: 0.7468
Epoch 22/55
10/10 - 0s - 8ms/step - accuracy: 0.6835 - loss: 0.8785 - val_accuracy: 0.7812 - val_loss: 0.7360
Epoch 23/55
10/10 - 0s - 7ms/step - accuracy: 0.6801 - loss: 0.7451 - val_accuracy: 0.7812 - val_loss: 0.7284
Epoch 24/55
10/10 - 0s - 8ms/step - accuracy: 0.7037 - loss: 0.7547 - val_accuracy: 0.7812 - val_loss: 0.7234
Epoch 25/55
10/10 - 0s - 8ms/step - accuracy: 0.6768 - loss: 0.6999 - val_accuracy: 0.7812 - val_loss: 0.7165
Epoch 26/55
10/10 - 0s - 8ms/step - accuracy: 0.7003 - loss: 0.7412 - val_accuracy: 0.7812 - val_loss: 0.7078
Epoch 27/55
10/10 - 0s - 9ms/step - accuracy: 0.7576 - loss: 0.6402 - val_accuracy: 0.7812 - val_loss: 0.6995
Epoch 28/55
10/10 - 0s - 9ms/step - accuracy: 0.7104 - loss: 0.7174 - val_accuracy: 0.7812 - val_loss: 0.6933
Epoch 29/55
10/10 - 0s - 8ms/step - accuracy: 0.6902 - loss: 0.7444 - val_accuracy: 0.7812 - val_loss: 0.6882
Epoch 30/55
10/10 - 0s - 8ms/step - accuracy: 0.7441 - loss: 0.6307 - val_accuracy: 0.7812 - val_loss: 0.6851
Epoch 31/55
10/10 - 0s - 8ms/step - accuracy: 0.7778 - loss: 0.5886 - val_accuracy: 0.7812 - val_loss: 0.6811
Epoch 32/55
10/10 - 0s - 7ms/step - accuracy: 0.7340 - loss: 0.6665 - val_accuracy: 0.7812 - val_loss: 0.6754
Epoch 33/55
10/10 - 0s - 8ms/step - accuracy: 0.7744 - loss: 0.5946 - val_accuracy: 0.7812 - val_loss: 0.6690
Epoch 34/55
10/10 - 0s - 9ms/step - accuracy: 0.7845 - loss: 0.6092 - val_accuracy: 0.7812 - val_loss: 0.6661
Epoch 35/55
10/10 - 0s - 8ms/step - accuracy: 0.7542 - loss: 0.6309 - val_accuracy: 0.7812 - val_loss: 0.6649
Epoch 36/55
10/10 - 0s - 8ms/step - accuracy: 0.7508 - loss: 0.6119 - val_accuracy: 0.7812 - val_loss: 0.6621
Epoch 37/55
10/10 - 0s - 9ms/step - accuracy: 0.7778 - loss: 0.6218 - val_accuracy: 0.7812 - val_loss: 0.6604
Epoch 38/55
10/10 - 0s - 9ms/step - accuracy: 0.7475 - loss: 0.5941 - val_accuracy: 0.7812 - val_loss: 0.6595
Epoch 39/55
10/10 - 0s - 9ms/step - accuracy: 0.7811 - loss: 0.5714 - val_accuracy: 0.7812 - val_loss: 0.6557
Epoch 40/55
10/10 - 0s - 8ms/step - accuracy: 0.7845 - loss: 0.5551 - val_accuracy: 0.7812 - val_loss: 0.6572
Epoch 41/55
10/10 - 0s - 8ms/step - accuracy: 0.7811 - loss: 0.5780 - val_accuracy: 0.7812 - val_loss: 0.6560
Epoch 42/55
10/10 - 0s - 8ms/step - accuracy: 0.7811 - loss: 0.5970 - val_accuracy: 0.7969 - val_loss: 0.6513
Epoch 43/55
10/10 - 0s - 8ms/step - accuracy: 0.7845 - loss: 0.5730 - val_accuracy: 0.7969 - val_loss: 0.6483
Epoch 44/55
10/10 - 0s - 8ms/step - accuracy: 0.8047 - loss: 0.5832 - val_accuracy: 0.7969 - val_loss: 0.6454
Epoch 45/55
10/10 - 0s - 8ms/step - accuracy: 0.8081 - loss: 0.5084 - val_accuracy: 0.7969 - val_loss: 0.6428
Epoch 46/55
10/10 - 0s - 8ms/step - accuracy: 0.7811 - loss: 0.5390 - val_accuracy: 0.7969 - val_loss: 0.6397
Epoch 47/55
10/10 - 0s - 8ms/step - accuracy: 0.7912 - loss: 0.5368 - val_accuracy: 0.7969 - val_loss: 0.6324
Epoch 48/55
10/10 - 0s - 8ms/step - accuracy: 0.7811 - loss: 0.5694 - val_accuracy: 0.7969 - val_loss: 0.6212
Epoch 49/55
10/10 - 0s - 9ms/step - accuracy: 0.8081 - loss: 0.5160 - val_accuracy: 0.7969 - val_loss: 0.6198
Epoch 50/55
10/10 - 0s - 9ms/step - accuracy: 0.7879 - loss: 0.5452 - val_accuracy: 0.7969 - val_loss: 0.6206
Epoch 51/55
10/10 - 0s - 8ms/step - accuracy: 0.7946 - loss: 0.5188 - val_accuracy: 0.7969 - val_loss: 0.6253
Epoch 52/55
10/10 - 0s - 8ms/step - accuracy: 0.8148 - loss: 0.4783 - val_accuracy: 0.7969 - val_loss: 0.6275
Epoch 53/55
10/10 - 0s - 8ms/step - accuracy: 0.8148 - loss: 0.4941 - val_accuracy: 0.7969 - val_loss: 0.6300
Epoch 54/55
10/10 - 0s - 8ms/step - accuracy: 0.8148 - loss: 0.4692 - val_accuracy: 0.7969 - val_loss: 0.6388
Epoch 55/55
10/10 - 0s - 8ms/step - accuracy: 0.8148 - loss: 0.4680 - val_accuracy: 0.7812 - val_loss: 0.6442
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 16ms/step
Precision: 0.6300
Recall: 0.7812
F1-Score: 0.6975

plot_confusion_matrices_nn(base_model_st, 
                        X_train_st, np.argmax(y_train, axis=1), 
                        X_val_st, np.argmax(y_val, axis=1), 
                        X_test_st, np.argmax(y_test, axis=1))

10/10 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 16ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 14ms/step

#Calculating different metrics on base_st data
base_metrics_st = model_performance_classification_sklearn_nn(base_model_st, 
                        X_train_st, np.argmax(y_train, axis=1), 
                        X_val_st, np.argmax(y_val, axis=1), 
                        X_test_st, np.argmax(y_test, axis=1))
print("Model performance:\n")
base_metrics_st

10/10 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step 
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 14ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 15ms/step
Model performance:

print("\nEvaluating tuned model:")
tuned_model_st, tuned_precision_st, tuned_recall_st, tuned_f1score_st = train_and_evaluate_model(X_train_st, X_val_st, y_train, y_val, use_tuned_model=True)

Evaluating tuned model:
X_train shape: (297, 384)
y_train shape: (297, 3)
Reloading Tuner from tuner_results\nn_tuning\tuner0.json

📌 Best Tuned Model Summary:

Model: "sequential_5"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Layer (type)                         ┃ Output Shape                ┃         Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ dense_15 (Dense)                     │ (None, 64)                  │          24,640 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_10               │ (None, 64)                  │             256 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_10 (Dropout)                 │ (None, 64)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_16 (Dense)                     │ (None, 16)                  │           1,040 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_11               │ (None, 16)                  │              64 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_11 (Dropout)                 │ (None, 16)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_17 (Dense)                     │ (None, 3)                   │              51 │
└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘

 Total params: 26,051 (101.76 KB)

 Trainable params: 25,891 (101.14 KB)

 Non-trainable params: 160 (640.00 B)

Epoch 1/55
10/10 - 1s - 148ms/step - accuracy: 0.3704 - loss: 1.6220 - val_accuracy: 0.5781 - val_loss: 1.0772
Epoch 2/55
10/10 - 0s - 8ms/step - accuracy: 0.3771 - loss: 1.4770 - val_accuracy: 0.7656 - val_loss: 1.0497
Epoch 3/55
10/10 - 0s - 8ms/step - accuracy: 0.4040 - loss: 1.4202 - val_accuracy: 0.7812 - val_loss: 1.0283
Epoch 4/55
10/10 - 0s - 9ms/step - accuracy: 0.4276 - loss: 1.3474 - val_accuracy: 0.7812 - val_loss: 1.0069
Epoch 5/55
10/10 - 0s - 8ms/step - accuracy: 0.4747 - loss: 1.2083 - val_accuracy: 0.7812 - val_loss: 0.9861
Epoch 6/55
10/10 - 0s - 8ms/step - accuracy: 0.5286 - loss: 1.0663 - val_accuracy: 0.7812 - val_loss: 0.9638
Epoch 7/55
10/10 - 0s - 8ms/step - accuracy: 0.5185 - loss: 1.0941 - val_accuracy: 0.7812 - val_loss: 0.9470
Epoch 8/55
10/10 - 0s - 8ms/step - accuracy: 0.5185 - loss: 1.0850 - val_accuracy: 0.7812 - val_loss: 0.9248
Epoch 9/55
10/10 - 0s - 9ms/step - accuracy: 0.5623 - loss: 0.9848 - val_accuracy: 0.7812 - val_loss: 0.9042
Epoch 10/55
10/10 - 0s - 8ms/step - accuracy: 0.5993 - loss: 0.9354 - val_accuracy: 0.7812 - val_loss: 0.8854
Epoch 11/55
10/10 - 0s - 8ms/step - accuracy: 0.6296 - loss: 0.8939 - val_accuracy: 0.7812 - val_loss: 0.8662
Epoch 12/55
10/10 - 0s - 8ms/step - accuracy: 0.6061 - loss: 0.8783 - val_accuracy: 0.7812 - val_loss: 0.8492
Epoch 13/55
10/10 - 0s - 8ms/step - accuracy: 0.6330 - loss: 0.8682 - val_accuracy: 0.7812 - val_loss: 0.8318
Epoch 14/55
10/10 - 0s - 9ms/step - accuracy: 0.6229 - loss: 0.8050 - val_accuracy: 0.7812 - val_loss: 0.8149
Epoch 15/55
10/10 - 0s - 9ms/step - accuracy: 0.6734 - loss: 0.7866 - val_accuracy: 0.7812 - val_loss: 0.8005
Epoch 16/55
10/10 - 0s - 9ms/step - accuracy: 0.6902 - loss: 0.7597 - val_accuracy: 0.7812 - val_loss: 0.7870
Epoch 17/55
10/10 - 0s - 8ms/step - accuracy: 0.7239 - loss: 0.6973 - val_accuracy: 0.7812 - val_loss: 0.7774
Epoch 18/55
10/10 - 0s - 8ms/step - accuracy: 0.7037 - loss: 0.7559 - val_accuracy: 0.7812 - val_loss: 0.7648
Epoch 19/55
10/10 - 0s - 9ms/step - accuracy: 0.7441 - loss: 0.6874 - val_accuracy: 0.7812 - val_loss: 0.7497
Epoch 20/55
10/10 - 0s - 9ms/step - accuracy: 0.7306 - loss: 0.7064 - val_accuracy: 0.7812 - val_loss: 0.7376
Epoch 21/55
10/10 - 0s - 8ms/step - accuracy: 0.7306 - loss: 0.6474 - val_accuracy: 0.7812 - val_loss: 0.7292
Epoch 22/55
10/10 - 0s - 8ms/step - accuracy: 0.7508 - loss: 0.6341 - val_accuracy: 0.7812 - val_loss: 0.7179
Epoch 23/55
10/10 - 0s - 8ms/step - accuracy: 0.7306 - loss: 0.6695 - val_accuracy: 0.7812 - val_loss: 0.7096
Epoch 24/55
10/10 - 0s - 8ms/step - accuracy: 0.7542 - loss: 0.5998 - val_accuracy: 0.7812 - val_loss: 0.7007
Epoch 25/55
10/10 - 0s - 8ms/step - accuracy: 0.8182 - loss: 0.5480 - val_accuracy: 0.7812 - val_loss: 0.6928
Epoch 26/55
10/10 - 0s - 9ms/step - accuracy: 0.7845 - loss: 0.5738 - val_accuracy: 0.7812 - val_loss: 0.6881
Epoch 27/55
10/10 - 0s - 9ms/step - accuracy: 0.7811 - loss: 0.6075 - val_accuracy: 0.7812 - val_loss: 0.6822
Epoch 28/55
10/10 - 0s - 8ms/step - accuracy: 0.7946 - loss: 0.5164 - val_accuracy: 0.7812 - val_loss: 0.6742
Epoch 29/55
10/10 - 0s - 8ms/step - accuracy: 0.8451 - loss: 0.4568 - val_accuracy: 0.7812 - val_loss: 0.6676
Epoch 30/55
10/10 - 0s - 8ms/step - accuracy: 0.8081 - loss: 0.5311 - val_accuracy: 0.7812 - val_loss: 0.6638
Epoch 31/55
10/10 - 0s - 8ms/step - accuracy: 0.8047 - loss: 0.5546 - val_accuracy: 0.7812 - val_loss: 0.6571
Epoch 32/55
10/10 - 0s - 8ms/step - accuracy: 0.8418 - loss: 0.4596 - val_accuracy: 0.7812 - val_loss: 0.6495
Epoch 33/55
10/10 - 0s - 8ms/step - accuracy: 0.8013 - loss: 0.5571 - val_accuracy: 0.7812 - val_loss: 0.6451
Epoch 34/55
10/10 - 0s - 8ms/step - accuracy: 0.8215 - loss: 0.4971 - val_accuracy: 0.7969 - val_loss: 0.6382
Epoch 35/55
10/10 - 0s - 8ms/step - accuracy: 0.8047 - loss: 0.4919 - val_accuracy: 0.7969 - val_loss: 0.6357
Epoch 36/55
10/10 - 0s - 8ms/step - accuracy: 0.8215 - loss: 0.4498 - val_accuracy: 0.7969 - val_loss: 0.6302
Epoch 37/55
10/10 - 0s - 8ms/step - accuracy: 0.8721 - loss: 0.3992 - val_accuracy: 0.7812 - val_loss: 0.6233
Epoch 38/55
10/10 - 0s - 8ms/step - accuracy: 0.8316 - loss: 0.4427 - val_accuracy: 0.7812 - val_loss: 0.6175
Epoch 39/55
10/10 - 0s - 8ms/step - accuracy: 0.8586 - loss: 0.3955 - val_accuracy: 0.7812 - val_loss: 0.6130
Epoch 40/55
10/10 - 0s - 8ms/step - accuracy: 0.8249 - loss: 0.4580 - val_accuracy: 0.7812 - val_loss: 0.6089
Epoch 41/55
10/10 - 0s - 8ms/step - accuracy: 0.8215 - loss: 0.4485 - val_accuracy: 0.7812 - val_loss: 0.6069
Epoch 42/55
10/10 - 0s - 8ms/step - accuracy: 0.8586 - loss: 0.4003 - val_accuracy: 0.7812 - val_loss: 0.6043
Epoch 43/55
10/10 - 0s - 8ms/step - accuracy: 0.8956 - loss: 0.3420 - val_accuracy: 0.7812 - val_loss: 0.6055
Epoch 44/55
10/10 - 0s - 8ms/step - accuracy: 0.8620 - loss: 0.3874 - val_accuracy: 0.7812 - val_loss: 0.6011
Epoch 45/55
10/10 - 0s - 8ms/step - accuracy: 0.8519 - loss: 0.3933 - val_accuracy: 0.7812 - val_loss: 0.5966
Epoch 46/55
10/10 - 0s - 8ms/step - accuracy: 0.8552 - loss: 0.4075 - val_accuracy: 0.7812 - val_loss: 0.5962
Epoch 47/55
10/10 - 0s - 8ms/step - accuracy: 0.8889 - loss: 0.3585 - val_accuracy: 0.7812 - val_loss: 0.5992
Epoch 48/55
10/10 - 0s - 8ms/step - accuracy: 0.8754 - loss: 0.3629 - val_accuracy: 0.7812 - val_loss: 0.5977
Epoch 49/55
10/10 - 0s - 8ms/step - accuracy: 0.8721 - loss: 0.3378 - val_accuracy: 0.7812 - val_loss: 0.5951
Epoch 50/55
10/10 - 0s - 8ms/step - accuracy: 0.8687 - loss: 0.3758 - val_accuracy: 0.7812 - val_loss: 0.5975
Epoch 51/55
10/10 - 0s - 8ms/step - accuracy: 0.8687 - loss: 0.3449 - val_accuracy: 0.7812 - val_loss: 0.6074
Epoch 52/55
10/10 - 0s - 8ms/step - accuracy: 0.8754 - loss: 0.3397 - val_accuracy: 0.7812 - val_loss: 0.6084
Epoch 53/55
10/10 - 0s - 8ms/step - accuracy: 0.8923 - loss: 0.3022 - val_accuracy: 0.7812 - val_loss: 0.6050
Epoch 54/55
10/10 - 0s - 9ms/step - accuracy: 0.8687 - loss: 0.3262 - val_accuracy: 0.7812 - val_loss: 0.6020
Epoch 55/55
10/10 - 0s - 8ms/step - accuracy: 0.8653 - loss: 0.3439 - val_accuracy: 0.7812 - val_loss: 0.6050
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 15ms/step
Precision: 0.6300
Recall: 0.7812
F1-Score: 0.6975

plot_confusion_matrices_nn(tuned_model_st, 
                        X_train_st, np.argmax(y_train, axis=1), 
                        X_val_st, np.argmax(y_val, axis=1), 
                        X_test_st, np.argmax(y_test, axis=1))

10/10 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 15ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 13ms/step

#Calculating different metrics on tuned_st data
tuned_metrics_st = model_performance_classification_sklearn_nn(tuned_model_st, 
                        X_train_st, np.argmax(y_train, axis=1), 
                        X_val_st, np.argmax(y_val, axis=1), 
                        X_test_st, np.argmax(y_test, axis=1))
print("Model performance:\n")
tuned_metrics_st

10/10 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step 
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 14ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 12ms/step
Model performance:

# Apply SMOTE to balance the dataset for Neural Networks
smote = SMOTE(sampling_strategy='auto', random_state=42)
X_train_wv_res, y_train_wv_res = smote.fit_resample(X_train_wv, y_train)  # Use np.argmax for class labels

X_train_gl_res, y_train_gl_res = smote.fit_resample(X_train_gl, y_train) 

X_train_st_res, y_train_st_res = smote.fit_resample(X_train_st, y_train)

  File "C:\Users\lakshman_kumar\anaconda3\Lib\site-packages\joblib\externals\loky\backend\context.py", line 257, in _count_physical_cores
    cpu_info = subprocess.run(
               ^^^^^^^^^^^^^^^
  File "C:\Users\lakshman_kumar\anaconda3\Lib\subprocess.py", line 548, in run
    with Popen(*popenargs, **kwargs) as process:
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\lakshman_kumar\anaconda3\Lib\subprocess.py", line 1026, in __init__
    self._execute_child(args, executable, preexec_fn, close_fds,
  File "C:\Users\lakshman_kumar\anaconda3\Lib\subprocess.py", line 1538, in _execute_child
    hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

print("Evaluating base model:")
base_model_wv_res, base_precision_wv_res, base_recall_wv_res, base_f1score_wv_res = train_and_evaluate_model(X_train_wv_res, X_val_wv, y_train_wv_res, y_val, use_tuned_model=False)

Evaluating base model:
X_train shape: (648, 300)
y_train shape: (648, 3)

📌 Base Model Summary:

Model: "sequential_6"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Layer (type)                         ┃ Output Shape                ┃         Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ dense_18 (Dense)                     │ (None, 32)                  │           9,632 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_12               │ (None, 32)                  │             128 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_12 (Dropout)                 │ (None, 32)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_19 (Dense)                     │ (None, 16)                  │             528 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_13               │ (None, 16)                  │              64 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_13 (Dropout)                 │ (None, 16)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_20 (Dense)                     │ (None, 3)                   │              51 │
└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘

 Total params: 10,403 (40.64 KB)

 Trainable params: 10,307 (40.26 KB)

 Non-trainable params: 96 (384.00 B)

Epoch 1/55
21/21 - 1s - 69ms/step - accuracy: 0.3488 - loss: 1.2370 - val_accuracy: 0.0781 - val_loss: 1.1041
Epoch 2/55
21/21 - 0s - 4ms/step - accuracy: 0.3966 - loss: 1.1308 - val_accuracy: 0.7812 - val_loss: 1.0920
Epoch 3/55
21/21 - 0s - 5ms/step - accuracy: 0.4105 - loss: 1.1035 - val_accuracy: 0.7812 - val_loss: 1.0906
Epoch 4/55
21/21 - 0s - 4ms/step - accuracy: 0.4275 - loss: 1.0704 - val_accuracy: 0.7812 - val_loss: 1.0885
Epoch 5/55
21/21 - 0s - 4ms/step - accuracy: 0.4321 - loss: 1.1014 - val_accuracy: 0.7812 - val_loss: 1.0903
Epoch 6/55
21/21 - 0s - 4ms/step - accuracy: 0.4043 - loss: 1.0794 - val_accuracy: 0.7812 - val_loss: 1.0883
Epoch 7/55
21/21 - 0s - 5ms/step - accuracy: 0.4275 - loss: 1.0643 - val_accuracy: 0.7656 - val_loss: 1.0960
Epoch 8/55
21/21 - 0s - 5ms/step - accuracy: 0.4568 - loss: 1.0557 - val_accuracy: 0.7656 - val_loss: 1.0953
Epoch 9/55
21/21 - 0s - 5ms/step - accuracy: 0.4537 - loss: 1.0362 - val_accuracy: 0.1406 - val_loss: 1.1014
Epoch 10/55
21/21 - 0s - 5ms/step - accuracy: 0.4352 - loss: 1.0485 - val_accuracy: 0.1406 - val_loss: 1.0993
Epoch 11/55
21/21 - 0s - 4ms/step - accuracy: 0.4491 - loss: 1.0308 - val_accuracy: 0.1406 - val_loss: 1.0903
Epoch 12/55
21/21 - 0s - 4ms/step - accuracy: 0.5154 - loss: 0.9983 - val_accuracy: 0.1406 - val_loss: 1.0966
Epoch 13/55
21/21 - 0s - 4ms/step - accuracy: 0.4799 - loss: 1.0003 - val_accuracy: 0.1406 - val_loss: 1.1114
Epoch 14/55
21/21 - 0s - 4ms/step - accuracy: 0.5525 - loss: 0.9505 - val_accuracy: 0.1406 - val_loss: 1.0941
Epoch 15/55
21/21 - 0s - 4ms/step - accuracy: 0.5571 - loss: 0.9468 - val_accuracy: 0.2188 - val_loss: 1.0651
Epoch 16/55
21/21 - 0s - 4ms/step - accuracy: 0.5895 - loss: 0.9132 - val_accuracy: 0.7031 - val_loss: 1.0782
Epoch 17/55
21/21 - 0s - 4ms/step - accuracy: 0.5880 - loss: 0.8909 - val_accuracy: 0.6719 - val_loss: 1.0811
Epoch 18/55
21/21 - 0s - 4ms/step - accuracy: 0.6049 - loss: 0.8610 - val_accuracy: 0.6406 - val_loss: 1.0666
Epoch 19/55
21/21 - 0s - 4ms/step - accuracy: 0.6281 - loss: 0.8045 - val_accuracy: 0.6094 - val_loss: 1.0513
Epoch 20/55
21/21 - 0s - 4ms/step - accuracy: 0.6049 - loss: 0.8554 - val_accuracy: 0.5625 - val_loss: 1.0680
Epoch 21/55
21/21 - 0s - 4ms/step - accuracy: 0.6219 - loss: 0.8196 - val_accuracy: 0.7188 - val_loss: 1.0234
Epoch 22/55
21/21 - 0s - 4ms/step - accuracy: 0.6451 - loss: 0.7868 - val_accuracy: 0.5938 - val_loss: 0.9814
Epoch 23/55
21/21 - 0s - 4ms/step - accuracy: 0.6636 - loss: 0.7646 - val_accuracy: 0.7344 - val_loss: 0.9701
Epoch 24/55
21/21 - 0s - 4ms/step - accuracy: 0.6157 - loss: 0.8231 - val_accuracy: 0.1562 - val_loss: 1.1264
Epoch 25/55
21/21 - 0s - 4ms/step - accuracy: 0.6559 - loss: 0.7655 - val_accuracy: 0.6406 - val_loss: 0.9937
Epoch 26/55
21/21 - 0s - 4ms/step - accuracy: 0.6651 - loss: 0.7463 - val_accuracy: 0.3906 - val_loss: 1.0342
Epoch 27/55
21/21 - 0s - 4ms/step - accuracy: 0.6867 - loss: 0.7243 - val_accuracy: 0.5000 - val_loss: 0.9364
Epoch 28/55
21/21 - 0s - 4ms/step - accuracy: 0.7114 - loss: 0.6967 - val_accuracy: 0.6719 - val_loss: 0.9456
Epoch 29/55
21/21 - 0s - 4ms/step - accuracy: 0.6821 - loss: 0.7240 - val_accuracy: 0.3438 - val_loss: 1.0568
Epoch 30/55
21/21 - 0s - 4ms/step - accuracy: 0.6960 - loss: 0.6814 - val_accuracy: 0.6406 - val_loss: 0.8430
Epoch 31/55
21/21 - 0s - 4ms/step - accuracy: 0.7284 - loss: 0.6405 - val_accuracy: 0.6094 - val_loss: 0.8210
Epoch 32/55
21/21 - 0s - 4ms/step - accuracy: 0.7052 - loss: 0.6518 - val_accuracy: 0.2500 - val_loss: 1.0725
Epoch 33/55
21/21 - 0s - 4ms/step - accuracy: 0.6991 - loss: 0.7047 - val_accuracy: 0.2656 - val_loss: 1.1264
Epoch 34/55
21/21 - 0s - 4ms/step - accuracy: 0.6960 - loss: 0.6601 - val_accuracy: 0.6719 - val_loss: 0.7264
Epoch 35/55
21/21 - 0s - 4ms/step - accuracy: 0.7330 - loss: 0.6420 - val_accuracy: 0.3438 - val_loss: 0.9246
Epoch 36/55
21/21 - 0s - 4ms/step - accuracy: 0.7022 - loss: 0.6432 - val_accuracy: 0.7031 - val_loss: 0.7626
Epoch 37/55
21/21 - 0s - 4ms/step - accuracy: 0.7330 - loss: 0.6041 - val_accuracy: 0.2344 - val_loss: 1.5890
Epoch 38/55
21/21 - 0s - 4ms/step - accuracy: 0.6960 - loss: 0.6687 - val_accuracy: 0.5000 - val_loss: 0.9725
Epoch 39/55
21/21 - 0s - 4ms/step - accuracy: 0.7577 - loss: 0.6026 - val_accuracy: 0.5469 - val_loss: 0.9925
Epoch 40/55
21/21 - 0s - 4ms/step - accuracy: 0.7454 - loss: 0.5948 - val_accuracy: 0.2500 - val_loss: 1.1360
Epoch 41/55
21/21 - 0s - 4ms/step - accuracy: 0.7145 - loss: 0.6478 - val_accuracy: 0.7031 - val_loss: 0.7534
Epoch 42/55
21/21 - 0s - 4ms/step - accuracy: 0.7253 - loss: 0.5994 - val_accuracy: 0.2344 - val_loss: 1.1714
Epoch 43/55
21/21 - 0s - 4ms/step - accuracy: 0.7824 - loss: 0.5566 - val_accuracy: 0.5156 - val_loss: 1.1547
Epoch 44/55
21/21 - 0s - 4ms/step - accuracy: 0.7515 - loss: 0.5797 - val_accuracy: 0.2656 - val_loss: 1.5479
Epoch 45/55
21/21 - 0s - 4ms/step - accuracy: 0.7840 - loss: 0.5327 - val_accuracy: 0.6562 - val_loss: 0.8668
Epoch 46/55
21/21 - 0s - 4ms/step - accuracy: 0.7747 - loss: 0.5344 - val_accuracy: 0.2500 - val_loss: 1.3995
Epoch 47/55
21/21 - 0s - 4ms/step - accuracy: 0.7701 - loss: 0.5949 - val_accuracy: 0.1719 - val_loss: 2.0380
Epoch 48/55
21/21 - 0s - 4ms/step - accuracy: 0.7500 - loss: 0.5816 - val_accuracy: 0.7031 - val_loss: 0.7853
Epoch 49/55
21/21 - 0s - 5ms/step - accuracy: 0.7731 - loss: 0.5213 - val_accuracy: 0.5469 - val_loss: 0.8426
Epoch 50/55
21/21 - 0s - 4ms/step - accuracy: 0.7392 - loss: 0.5818 - val_accuracy: 0.1094 - val_loss: 4.3287
Epoch 51/55
21/21 - 0s - 4ms/step - accuracy: 0.7639 - loss: 0.5615 - val_accuracy: 0.6875 - val_loss: 0.7383
Epoch 52/55
21/21 - 0s - 4ms/step - accuracy: 0.7670 - loss: 0.5399 - val_accuracy: 0.2969 - val_loss: 1.2541
Epoch 53/55
21/21 - 0s - 4ms/step - accuracy: 0.7731 - loss: 0.5350 - val_accuracy: 0.1406 - val_loss: 2.3289
Epoch 54/55
21/21 - 0s - 4ms/step - accuracy: 0.7623 - loss: 0.5601 - val_accuracy: 0.2812 - val_loss: 1.3577
Epoch 55/55
21/21 - 0s - 4ms/step - accuracy: 0.7778 - loss: 0.5186 - val_accuracy: 0.3594 - val_loss: 1.0144
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 12ms/step
Precision: 0.6496
Recall: 0.3594
F1-Score: 0.3973

plot_confusion_matrices_nn(base_model_wv_res, 
                        X_train_wv_res, np.argmax(y_train_wv_res, axis=1), 
                        X_val_wv, np.argmax(y_val, axis=1), 
                        X_test_wv, np.argmax(y_test, axis=1))

21/21 ━━━━━━━━━━━━━━━━━━━━ 0s 4ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 14ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 12ms/step

#Calculating different metrics on base_wv data
base_metrics_wv_res = model_performance_classification_sklearn_nn(base_model_wv_res, 
                        X_train_wv_res, np.argmax(y_train_wv_res, axis=1), 
                        X_val_wv, np.argmax(y_val, axis=1), 
                        X_test_wv, np.argmax(y_test, axis=1))
print("Model performance:\n")
base_metrics_wv_res

21/21 ━━━━━━━━━━━━━━━━━━━━ 0s 1ms/step 
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 12ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 12ms/step
Model performance:

print("\nEvaluating tuned model:")
tuned_model_wv_res, tuned_precision_wv_res, tuned_recall_wv_res, tuned_f1score_wv_res = train_and_evaluate_model(X_train_wv_res, X_val_wv, y_train_wv_res, y_val, use_tuned_model=True)

Evaluating tuned model:
X_train shape: (648, 300)
y_train shape: (648, 3)
Reloading Tuner from tuner_results\nn_tuning\tuner0.json

📌 Best Tuned Model Summary:

Model: "sequential_7"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Layer (type)                         ┃ Output Shape                ┃         Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ dense_21 (Dense)                     │ (None, 32)                  │           9,632 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_14               │ (None, 32)                  │             128 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_14 (Dropout)                 │ (None, 32)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_22 (Dense)                     │ (None, 64)                  │           2,112 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_15               │ (None, 64)                  │             256 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_15 (Dropout)                 │ (None, 64)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_23 (Dense)                     │ (None, 3)                   │             195 │
└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘

 Total params: 12,323 (48.14 KB)

 Trainable params: 12,131 (47.39 KB)

 Non-trainable params: 192 (768.00 B)

Epoch 1/55
21/21 - 2s - 76ms/step - accuracy: 0.3580 - loss: 1.2227 - val_accuracy: 0.1406 - val_loss: 1.0901
Epoch 2/55
21/21 - 0s - 5ms/step - accuracy: 0.4043 - loss: 1.1225 - val_accuracy: 0.1406 - val_loss: 1.0846
Epoch 3/55
21/21 - 0s - 4ms/step - accuracy: 0.4491 - loss: 1.0531 - val_accuracy: 0.7812 - val_loss: 1.0861
Epoch 4/55
21/21 - 0s - 4ms/step - accuracy: 0.4954 - loss: 1.0367 - val_accuracy: 0.1406 - val_loss: 1.0980
Epoch 5/55
21/21 - 0s - 5ms/step - accuracy: 0.4985 - loss: 1.0036 - val_accuracy: 0.2656 - val_loss: 1.0914
Epoch 6/55
21/21 - 0s - 5ms/step - accuracy: 0.5370 - loss: 0.9560 - val_accuracy: 0.7812 - val_loss: 1.0759
Epoch 7/55
21/21 - 0s - 4ms/step - accuracy: 0.5617 - loss: 0.9326 - val_accuracy: 0.7656 - val_loss: 1.0714
Epoch 8/55
21/21 - 0s - 5ms/step - accuracy: 0.5910 - loss: 0.9015 - val_accuracy: 0.7812 - val_loss: 1.0644
Epoch 9/55
21/21 - 0s - 5ms/step - accuracy: 0.6790 - loss: 0.7900 - val_accuracy: 0.7812 - val_loss: 1.0521
Epoch 10/55
21/21 - 0s - 5ms/step - accuracy: 0.6698 - loss: 0.7851 - val_accuracy: 0.7812 - val_loss: 1.0336
Epoch 11/55
21/21 - 0s - 5ms/step - accuracy: 0.6744 - loss: 0.7625 - val_accuracy: 0.7812 - val_loss: 1.0418
Epoch 12/55
21/21 - 0s - 4ms/step - accuracy: 0.6667 - loss: 0.7466 - val_accuracy: 0.7812 - val_loss: 1.0147
Epoch 13/55
21/21 - 0s - 4ms/step - accuracy: 0.7099 - loss: 0.6865 - val_accuracy: 0.7812 - val_loss: 0.9631
Epoch 14/55
21/21 - 0s - 5ms/step - accuracy: 0.7191 - loss: 0.6877 - val_accuracy: 0.7812 - val_loss: 0.9342
Epoch 15/55
21/21 - 0s - 5ms/step - accuracy: 0.7423 - loss: 0.6268 - val_accuracy: 0.7812 - val_loss: 0.9188
Epoch 16/55
21/21 - 0s - 4ms/step - accuracy: 0.7469 - loss: 0.6360 - val_accuracy: 0.7812 - val_loss: 0.9439
Epoch 17/55
21/21 - 0s - 4ms/step - accuracy: 0.7438 - loss: 0.6061 - val_accuracy: 0.7812 - val_loss: 0.9647
Epoch 18/55
21/21 - 0s - 4ms/step - accuracy: 0.7531 - loss: 0.6024 - val_accuracy: 0.7812 - val_loss: 0.9722
Epoch 19/55
21/21 - 0s - 4ms/step - accuracy: 0.7546 - loss: 0.5790 - val_accuracy: 0.7812 - val_loss: 0.9159
Epoch 20/55
21/21 - 0s - 4ms/step - accuracy: 0.7593 - loss: 0.5544 - val_accuracy: 0.7812 - val_loss: 0.9546
Epoch 21/55
21/21 - 0s - 4ms/step - accuracy: 0.7963 - loss: 0.5397 - val_accuracy: 0.7812 - val_loss: 0.9598
Epoch 22/55
21/21 - 0s - 4ms/step - accuracy: 0.7948 - loss: 0.5191 - val_accuracy: 0.3594 - val_loss: 1.0184
Epoch 23/55
21/21 - 0s - 4ms/step - accuracy: 0.7685 - loss: 0.5449 - val_accuracy: 0.5781 - val_loss: 0.9777
Epoch 24/55
21/21 - 0s - 4ms/step - accuracy: 0.7793 - loss: 0.5460 - val_accuracy: 0.5469 - val_loss: 0.9567
Epoch 25/55
21/21 - 0s - 4ms/step - accuracy: 0.8071 - loss: 0.4685 - val_accuracy: 0.6719 - val_loss: 0.9120
Epoch 26/55
21/21 - 0s - 5ms/step - accuracy: 0.8086 - loss: 0.4855 - val_accuracy: 0.6250 - val_loss: 0.9076
Epoch 27/55
21/21 - 0s - 5ms/step - accuracy: 0.8287 - loss: 0.4314 - val_accuracy: 0.5781 - val_loss: 0.8613
Epoch 28/55
21/21 - 0s - 5ms/step - accuracy: 0.8164 - loss: 0.4452 - val_accuracy: 0.1250 - val_loss: 1.4643
Epoch 29/55
21/21 - 0s - 6ms/step - accuracy: 0.8302 - loss: 0.4331 - val_accuracy: 0.1094 - val_loss: 1.7455
Epoch 30/55
21/21 - 0s - 6ms/step - accuracy: 0.8256 - loss: 0.4548 - val_accuracy: 0.6719 - val_loss: 0.8976
Epoch 31/55
21/21 - 0s - 4ms/step - accuracy: 0.7994 - loss: 0.4812 - val_accuracy: 0.5469 - val_loss: 0.9048
Epoch 32/55
21/21 - 0s - 5ms/step - accuracy: 0.8318 - loss: 0.4499 - val_accuracy: 0.5625 - val_loss: 0.8719
Epoch 33/55
21/21 - 0s - 5ms/step - accuracy: 0.8241 - loss: 0.4127 - val_accuracy: 0.1719 - val_loss: 1.7349
Epoch 34/55
21/21 - 0s - 5ms/step - accuracy: 0.8534 - loss: 0.4095 - val_accuracy: 0.3750 - val_loss: 1.5623
Epoch 35/55
21/21 - 0s - 5ms/step - accuracy: 0.8040 - loss: 0.4641 - val_accuracy: 0.3125 - val_loss: 1.8550
Epoch 36/55
21/21 - 0s - 4ms/step - accuracy: 0.8179 - loss: 0.4287 - val_accuracy: 0.6250 - val_loss: 1.1765
Epoch 37/55
21/21 - 0s - 4ms/step - accuracy: 0.8519 - loss: 0.3711 - val_accuracy: 0.7812 - val_loss: 1.2663
Epoch 38/55
21/21 - 0s - 4ms/step - accuracy: 0.8704 - loss: 0.3392 - val_accuracy: 0.7812 - val_loss: 1.2260
Epoch 39/55
21/21 - 0s - 4ms/step - accuracy: 0.8488 - loss: 0.4003 - val_accuracy: 0.4375 - val_loss: 1.4729
Epoch 40/55
21/21 - 0s - 4ms/step - accuracy: 0.8472 - loss: 0.3770 - val_accuracy: 0.7188 - val_loss: 0.9295
Epoch 41/55
21/21 - 0s - 4ms/step - accuracy: 0.8673 - loss: 0.3542 - val_accuracy: 0.2031 - val_loss: 2.3635
Epoch 42/55
21/21 - 0s - 5ms/step - accuracy: 0.8627 - loss: 0.3411 - val_accuracy: 0.4062 - val_loss: 1.2553
Epoch 43/55
21/21 - 0s - 5ms/step - accuracy: 0.8441 - loss: 0.3566 - val_accuracy: 0.1719 - val_loss: 3.5327
Epoch 44/55
21/21 - 0s - 4ms/step - accuracy: 0.8750 - loss: 0.3275 - val_accuracy: 0.6250 - val_loss: 0.9949
Epoch 45/55
21/21 - 0s - 4ms/step - accuracy: 0.8750 - loss: 0.3407 - val_accuracy: 0.4219 - val_loss: 1.7045
Epoch 46/55
21/21 - 0s - 4ms/step - accuracy: 0.8580 - loss: 0.3673 - val_accuracy: 0.3125 - val_loss: 2.2611
Epoch 47/55
21/21 - 0s - 5ms/step - accuracy: 0.8503 - loss: 0.3773 - val_accuracy: 0.0781 - val_loss: 9.8680
Epoch 48/55
21/21 - 0s - 4ms/step - accuracy: 0.8642 - loss: 0.3465 - val_accuracy: 0.0781 - val_loss: 10.2249
Epoch 49/55
21/21 - 0s - 5ms/step - accuracy: 0.8472 - loss: 0.3733 - val_accuracy: 0.0781 - val_loss: 9.9812
Epoch 50/55
21/21 - 0s - 5ms/step - accuracy: 0.8627 - loss: 0.3491 - val_accuracy: 0.0781 - val_loss: 8.7583
Epoch 51/55
21/21 - 0s - 5ms/step - accuracy: 0.8920 - loss: 0.3231 - val_accuracy: 0.1094 - val_loss: 4.8427
Epoch 52/55
21/21 - 0s - 5ms/step - accuracy: 0.8735 - loss: 0.3231 - val_accuracy: 0.0781 - val_loss: 7.3631
Epoch 53/55
21/21 - 0s - 5ms/step - accuracy: 0.8843 - loss: 0.3120 - val_accuracy: 0.1094 - val_loss: 4.4431
Epoch 54/55
21/21 - 0s - 5ms/step - accuracy: 0.8873 - loss: 0.2841 - val_accuracy: 0.2344 - val_loss: 3.4816
Epoch 55/55
21/21 - 0s - 5ms/step - accuracy: 0.8981 - loss: 0.2675 - val_accuracy: 0.1094 - val_loss: 5.2459
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 13ms/step
Precision: 0.5272
Recall: 0.1094
F1-Score: 0.0708

plot_confusion_matrices_nn(tuned_model_wv_res, 
                        X_train_wv_res, np.argmax(y_train_wv_res, axis=1), 
                        X_val_wv, np.argmax(y_val, axis=1), 
                        X_test_wv, np.argmax(y_test, axis=1))

21/21 ━━━━━━━━━━━━━━━━━━━━ 0s 4ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 12ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 18ms/step

#Calculating different metrics on tuned_wv data
tuned_metrics_wv_res = model_performance_classification_sklearn_nn(tuned_model_wv_res, 
                        X_train_wv_res, np.argmax(y_train_wv_res, axis=1), 
                        X_val_wv, np.argmax(y_val, axis=1), 
                        X_test_wv, np.argmax(y_test, axis=1))
print("Model performance:\n")
tuned_metrics_wv_res

21/21 ━━━━━━━━━━━━━━━━━━━━ 0s 1ms/step 
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 13ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 10ms/step
Model performance:

print("Evaluating base model:")
base_model_gl_res, base_precision_gl_res, base_recall_gl_res, base_f1score_gl_res = train_and_evaluate_model(X_train_gl_res, X_val_gl, y_train_gl_res, y_val, use_tuned_model=False)

Evaluating base model:
X_train shape: (648, 100)
y_train shape: (648, 3)

📌 Base Model Summary:

Model: "sequential_8"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Layer (type)                         ┃ Output Shape                ┃         Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ dense_24 (Dense)                     │ (None, 32)                  │           3,232 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_16               │ (None, 32)                  │             128 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_16 (Dropout)                 │ (None, 32)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_25 (Dense)                     │ (None, 16)                  │             528 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_17               │ (None, 16)                  │              64 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_17 (Dropout)                 │ (None, 16)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_26 (Dense)                     │ (None, 3)                   │              51 │
└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘

 Total params: 4,003 (15.64 KB)

 Trainable params: 3,907 (15.26 KB)

 Non-trainable params: 96 (384.00 B)

Epoch 1/55
21/21 - 2s - 76ms/step - accuracy: 0.2901 - loss: 1.7562 - val_accuracy: 0.1250 - val_loss: 1.3511
Epoch 2/55
21/21 - 0s - 4ms/step - accuracy: 0.3688 - loss: 1.5398 - val_accuracy: 0.1406 - val_loss: 1.2639
Epoch 3/55
21/21 - 0s - 5ms/step - accuracy: 0.3549 - loss: 1.4338 - val_accuracy: 0.1406 - val_loss: 1.2001
Epoch 4/55
21/21 - 0s - 5ms/step - accuracy: 0.3673 - loss: 1.3815 - val_accuracy: 0.1406 - val_loss: 1.1518
Epoch 5/55
21/21 - 0s - 5ms/step - accuracy: 0.4460 - loss: 1.2632 - val_accuracy: 0.1562 - val_loss: 1.1232
Epoch 6/55
21/21 - 0s - 5ms/step - accuracy: 0.3858 - loss: 1.2574 - val_accuracy: 0.2188 - val_loss: 1.0938
Epoch 7/55
21/21 - 0s - 5ms/step - accuracy: 0.4491 - loss: 1.1591 - val_accuracy: 0.2969 - val_loss: 1.0568
Epoch 8/55
21/21 - 0s - 4ms/step - accuracy: 0.4028 - loss: 1.1824 - val_accuracy: 0.3281 - val_loss: 1.0473
Epoch 9/55
21/21 - 0s - 5ms/step - accuracy: 0.4429 - loss: 1.1408 - val_accuracy: 0.2656 - val_loss: 1.0472
Epoch 10/55
21/21 - 0s - 4ms/step - accuracy: 0.4614 - loss: 1.1224 - val_accuracy: 0.2344 - val_loss: 1.0475
Epoch 11/55
21/21 - 0s - 4ms/step - accuracy: 0.4676 - loss: 1.0739 - val_accuracy: 0.2500 - val_loss: 1.0545
Epoch 12/55
21/21 - 0s - 4ms/step - accuracy: 0.5093 - loss: 1.0168 - val_accuracy: 0.2344 - val_loss: 1.0318
Epoch 13/55
21/21 - 0s - 4ms/step - accuracy: 0.5139 - loss: 1.0076 - val_accuracy: 0.2656 - val_loss: 1.0572
Epoch 14/55
21/21 - 0s - 4ms/step - accuracy: 0.5108 - loss: 1.0279 - val_accuracy: 0.2656 - val_loss: 1.0672
Epoch 15/55
21/21 - 0s - 4ms/step - accuracy: 0.5108 - loss: 0.9631 - val_accuracy: 0.2812 - val_loss: 1.0634
Epoch 16/55
21/21 - 0s - 4ms/step - accuracy: 0.5201 - loss: 0.9766 - val_accuracy: 0.2812 - val_loss: 1.0574
Epoch 17/55
21/21 - 0s - 4ms/step - accuracy: 0.5525 - loss: 0.9166 - val_accuracy: 0.2656 - val_loss: 1.0687
Epoch 18/55
21/21 - 0s - 4ms/step - accuracy: 0.5417 - loss: 0.9589 - val_accuracy: 0.3281 - val_loss: 1.0337
Epoch 19/55
21/21 - 0s - 4ms/step - accuracy: 0.5540 - loss: 0.9437 - val_accuracy: 0.3594 - val_loss: 1.0084
Epoch 20/55
21/21 - 0s - 4ms/step - accuracy: 0.5926 - loss: 0.8609 - val_accuracy: 0.3281 - val_loss: 1.0207
Epoch 21/55
21/21 - 0s - 4ms/step - accuracy: 0.5694 - loss: 0.9427 - val_accuracy: 0.3438 - val_loss: 1.0004
Epoch 22/55
21/21 - 0s - 4ms/step - accuracy: 0.6219 - loss: 0.8749 - val_accuracy: 0.4062 - val_loss: 0.9972
Epoch 23/55
21/21 - 0s - 4ms/step - accuracy: 0.5802 - loss: 0.8712 - val_accuracy: 0.3438 - val_loss: 1.0401
Epoch 24/55
21/21 - 0s - 5ms/step - accuracy: 0.5818 - loss: 0.8994 - val_accuracy: 0.3281 - val_loss: 1.0759
Epoch 25/55
21/21 - 0s - 4ms/step - accuracy: 0.6188 - loss: 0.8389 - val_accuracy: 0.3281 - val_loss: 1.0859
Epoch 26/55
21/21 - 0s - 5ms/step - accuracy: 0.6204 - loss: 0.8669 - val_accuracy: 0.3750 - val_loss: 1.0221
Epoch 27/55
21/21 - 0s - 6ms/step - accuracy: 0.6080 - loss: 0.8458 - val_accuracy: 0.4219 - val_loss: 0.9932
Epoch 28/55
21/21 - 0s - 5ms/step - accuracy: 0.6065 - loss: 0.8182 - val_accuracy: 0.4844 - val_loss: 0.8988
Epoch 29/55
21/21 - 0s - 5ms/step - accuracy: 0.6265 - loss: 0.8286 - val_accuracy: 0.4688 - val_loss: 0.9576
Epoch 30/55
21/21 - 0s - 5ms/step - accuracy: 0.6420 - loss: 0.8037 - val_accuracy: 0.4688 - val_loss: 0.9981
Epoch 31/55
21/21 - 0s - 5ms/step - accuracy: 0.6667 - loss: 0.7755 - val_accuracy: 0.4375 - val_loss: 1.0239
Epoch 32/55
21/21 - 0s - 5ms/step - accuracy: 0.6559 - loss: 0.8016 - val_accuracy: 0.4531 - val_loss: 0.9647
Epoch 33/55
21/21 - 0s - 5ms/step - accuracy: 0.6698 - loss: 0.7636 - val_accuracy: 0.4844 - val_loss: 0.8954
Epoch 34/55
21/21 - 0s - 5ms/step - accuracy: 0.6451 - loss: 0.7775 - val_accuracy: 0.4844 - val_loss: 0.9533
Epoch 35/55
21/21 - 0s - 5ms/step - accuracy: 0.6790 - loss: 0.7664 - val_accuracy: 0.5312 - val_loss: 0.9161
Epoch 36/55
21/21 - 0s - 5ms/step - accuracy: 0.6775 - loss: 0.7309 - val_accuracy: 0.4844 - val_loss: 0.9440
Epoch 37/55
21/21 - 0s - 4ms/step - accuracy: 0.6898 - loss: 0.7161 - val_accuracy: 0.4844 - val_loss: 0.9431
Epoch 38/55
21/21 - 0s - 4ms/step - accuracy: 0.6883 - loss: 0.7240 - val_accuracy: 0.5156 - val_loss: 0.9428
Epoch 39/55
21/21 - 0s - 4ms/step - accuracy: 0.7083 - loss: 0.7078 - val_accuracy: 0.4375 - val_loss: 0.9856
Epoch 40/55
21/21 - 0s - 4ms/step - accuracy: 0.6944 - loss: 0.7235 - val_accuracy: 0.5312 - val_loss: 0.9246
Epoch 41/55
21/21 - 0s - 5ms/step - accuracy: 0.6929 - loss: 0.7279 - val_accuracy: 0.5156 - val_loss: 0.9265
Epoch 42/55
21/21 - 0s - 4ms/step - accuracy: 0.6960 - loss: 0.7106 - val_accuracy: 0.5312 - val_loss: 0.8673
Epoch 43/55
21/21 - 0s - 4ms/step - accuracy: 0.6991 - loss: 0.7014 - val_accuracy: 0.5000 - val_loss: 0.8909
Epoch 44/55
21/21 - 0s - 4ms/step - accuracy: 0.7269 - loss: 0.6766 - val_accuracy: 0.5156 - val_loss: 0.8726
Epoch 45/55
21/21 - 0s - 4ms/step - accuracy: 0.7670 - loss: 0.6239 - val_accuracy: 0.4531 - val_loss: 0.8677
Epoch 46/55
21/21 - 0s - 4ms/step - accuracy: 0.7546 - loss: 0.6338 - val_accuracy: 0.4688 - val_loss: 0.8723
Epoch 47/55
21/21 - 0s - 5ms/step - accuracy: 0.7515 - loss: 0.5900 - val_accuracy: 0.4688 - val_loss: 0.8867
Epoch 48/55
21/21 - 0s - 5ms/step - accuracy: 0.7438 - loss: 0.6378 - val_accuracy: 0.5625 - val_loss: 0.8260
Epoch 49/55
21/21 - 0s - 5ms/step - accuracy: 0.7701 - loss: 0.6083 - val_accuracy: 0.4844 - val_loss: 0.8203
Epoch 50/55
21/21 - 0s - 5ms/step - accuracy: 0.7546 - loss: 0.6101 - val_accuracy: 0.5312 - val_loss: 0.7783
Epoch 51/55
21/21 - 0s - 5ms/step - accuracy: 0.7654 - loss: 0.6006 - val_accuracy: 0.6562 - val_loss: 0.6944
Epoch 52/55
21/21 - 0s - 5ms/step - accuracy: 0.7716 - loss: 0.5815 - val_accuracy: 0.5156 - val_loss: 0.8512
Epoch 53/55
21/21 - 0s - 5ms/step - accuracy: 0.7577 - loss: 0.5830 - val_accuracy: 0.4844 - val_loss: 0.8226
Epoch 54/55
21/21 - 0s - 5ms/step - accuracy: 0.7793 - loss: 0.5764 - val_accuracy: 0.5156 - val_loss: 0.8688
Epoch 55/55
21/21 - 0s - 4ms/step - accuracy: 0.7377 - loss: 0.6014 - val_accuracy: 0.5625 - val_loss: 0.8518
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 14ms/step
Precision: 0.7192
Recall: 0.5625
F1-Score: 0.6104

plot_confusion_matrices_nn(base_model_gl_res, 
                        X_train_gl_res, np.argmax(y_train_gl_res, axis=1), 
                        X_val_gl, np.argmax(y_val, axis=1), 
                        X_test_gl, np.argmax(y_test, axis=1))

21/21 ━━━━━━━━━━━━━━━━━━━━ 0s 3ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 11ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 13ms/step

#Calculating different metrics on base_gl data
base_metrics_gl_res = model_performance_classification_sklearn_nn(base_model_gl_res, 
                        X_train_gl_res, np.argmax(y_train_gl_res, axis=1), 
                        X_val_gl, np.argmax(y_val, axis=1), 
                        X_test_gl, np.argmax(y_test, axis=1))
print("Model performance:\n")
base_metrics_gl_res

21/21 ━━━━━━━━━━━━━━━━━━━━ 0s 1ms/step 
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 13ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 14ms/step
Model performance:

print("\nEvaluating tuned model:")
tuned_model_gl_res, tuned_precision_gl_res, tuned_recall_gl_res, tuned_f1score_gl_res = train_and_evaluate_model(X_train_gl_res, X_val_gl, y_train_gl_res, y_val, use_tuned_model=True)

Evaluating tuned model:
X_train shape: (648, 100)
y_train shape: (648, 3)
Reloading Tuner from tuner_results\nn_tuning\tuner0.json

📌 Best Tuned Model Summary:

Model: "sequential_9"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Layer (type)                         ┃ Output Shape                ┃         Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ dense_27 (Dense)                     │ (None, 64)                  │           6,464 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_18               │ (None, 64)                  │             256 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_18 (Dropout)                 │ (None, 64)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_28 (Dense)                     │ (None, 16)                  │           1,040 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_19               │ (None, 16)                  │              64 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_19 (Dropout)                 │ (None, 16)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_29 (Dense)                     │ (None, 3)                   │              51 │
└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘

 Total params: 7,875 (30.76 KB)

 Trainable params: 7,715 (30.14 KB)

 Non-trainable params: 160 (640.00 B)

Epoch 1/55
21/21 - 2s - 81ms/step - accuracy: 0.4398 - loss: 1.2889 - val_accuracy: 0.1875 - val_loss: 1.0752
Epoch 2/55
21/21 - 0s - 6ms/step - accuracy: 0.5386 - loss: 1.0204 - val_accuracy: 0.2031 - val_loss: 1.0816
Epoch 3/55
21/21 - 0s - 5ms/step - accuracy: 0.5849 - loss: 0.9227 - val_accuracy: 0.2344 - val_loss: 1.1108
Epoch 4/55
21/21 - 0s - 5ms/step - accuracy: 0.6312 - loss: 0.8200 - val_accuracy: 0.5781 - val_loss: 0.9777
Epoch 5/55
21/21 - 0s - 5ms/step - accuracy: 0.6451 - loss: 0.7771 - val_accuracy: 0.2812 - val_loss: 1.1269
Epoch 6/55
21/21 - 0s - 5ms/step - accuracy: 0.6775 - loss: 0.7609 - val_accuracy: 0.5312 - val_loss: 0.8724
Epoch 7/55
21/21 - 0s - 5ms/step - accuracy: 0.7145 - loss: 0.7180 - val_accuracy: 0.4062 - val_loss: 0.9627
Epoch 8/55
21/21 - 0s - 4ms/step - accuracy: 0.7037 - loss: 0.6996 - val_accuracy: 0.5469 - val_loss: 0.8652
Epoch 9/55
21/21 - 0s - 4ms/step - accuracy: 0.7562 - loss: 0.6241 - val_accuracy: 0.5469 - val_loss: 0.8867
Epoch 10/55
21/21 - 0s - 5ms/step - accuracy: 0.7593 - loss: 0.6145 - val_accuracy: 0.6562 - val_loss: 0.7585
Epoch 11/55
21/21 - 0s - 6ms/step - accuracy: 0.7423 - loss: 0.6242 - val_accuracy: 0.6250 - val_loss: 0.7265
Epoch 12/55
21/21 - 0s - 6ms/step - accuracy: 0.7731 - loss: 0.5720 - val_accuracy: 0.5312 - val_loss: 0.9939
Epoch 13/55
21/21 - 0s - 5ms/step - accuracy: 0.7932 - loss: 0.5231 - val_accuracy: 0.7031 - val_loss: 0.8191
Epoch 14/55
21/21 - 0s - 5ms/step - accuracy: 0.7562 - loss: 0.6044 - val_accuracy: 0.6250 - val_loss: 0.9531
Epoch 15/55
21/21 - 0s - 5ms/step - accuracy: 0.7932 - loss: 0.5439 - val_accuracy: 0.6875 - val_loss: 0.9955
Epoch 16/55
21/21 - 0s - 5ms/step - accuracy: 0.7608 - loss: 0.5765 - val_accuracy: 0.7031 - val_loss: 0.8634
Epoch 17/55
21/21 - 0s - 4ms/step - accuracy: 0.8272 - loss: 0.5022 - val_accuracy: 0.4531 - val_loss: 1.2598
Epoch 18/55
21/21 - 0s - 4ms/step - accuracy: 0.7716 - loss: 0.5507 - val_accuracy: 0.5781 - val_loss: 1.1617
Epoch 19/55
21/21 - 0s - 5ms/step - accuracy: 0.7948 - loss: 0.5353 - val_accuracy: 0.4844 - val_loss: 1.1072
Epoch 20/55
21/21 - 0s - 5ms/step - accuracy: 0.8117 - loss: 0.5078 - val_accuracy: 0.5781 - val_loss: 1.0924
Epoch 21/55
21/21 - 0s - 5ms/step - accuracy: 0.7994 - loss: 0.5144 - val_accuracy: 0.6250 - val_loss: 1.1500
Epoch 22/55
21/21 - 0s - 4ms/step - accuracy: 0.7778 - loss: 0.5336 - val_accuracy: 0.4375 - val_loss: 1.6301
Epoch 23/55
21/21 - 0s - 4ms/step - accuracy: 0.8117 - loss: 0.4799 - val_accuracy: 0.6562 - val_loss: 1.0757
Epoch 24/55
21/21 - 0s - 5ms/step - accuracy: 0.8117 - loss: 0.5057 - val_accuracy: 0.5781 - val_loss: 1.1776
Epoch 25/55
21/21 - 0s - 5ms/step - accuracy: 0.8194 - loss: 0.4621 - val_accuracy: 0.6250 - val_loss: 1.1631
Epoch 26/55
21/21 - 0s - 4ms/step - accuracy: 0.8333 - loss: 0.4668 - val_accuracy: 0.5000 - val_loss: 1.3883
Epoch 27/55
21/21 - 0s - 4ms/step - accuracy: 0.8117 - loss: 0.5045 - val_accuracy: 0.5000 - val_loss: 1.3563
Epoch 28/55
21/21 - 0s - 4ms/step - accuracy: 0.8225 - loss: 0.4658 - val_accuracy: 0.5625 - val_loss: 1.2609
Epoch 29/55
21/21 - 0s - 6ms/step - accuracy: 0.8318 - loss: 0.4691 - val_accuracy: 0.5000 - val_loss: 1.3931
Epoch 30/55
21/21 - 0s - 6ms/step - accuracy: 0.8179 - loss: 0.4905 - val_accuracy: 0.4844 - val_loss: 1.4622
Epoch 31/55
21/21 - 0s - 4ms/step - accuracy: 0.8380 - loss: 0.4566 - val_accuracy: 0.4375 - val_loss: 1.7613
Epoch 32/55
21/21 - 0s - 4ms/step - accuracy: 0.8117 - loss: 0.4992 - val_accuracy: 0.5781 - val_loss: 1.2540
Epoch 33/55
21/21 - 0s - 5ms/step - accuracy: 0.8302 - loss: 0.4364 - val_accuracy: 0.5469 - val_loss: 1.4716
Epoch 34/55
21/21 - 0s - 5ms/step - accuracy: 0.8380 - loss: 0.4493 - val_accuracy: 0.5156 - val_loss: 1.5106
Epoch 35/55
21/21 - 0s - 5ms/step - accuracy: 0.8426 - loss: 0.4230 - val_accuracy: 0.5312 - val_loss: 1.4901
Epoch 36/55
21/21 - 0s - 5ms/step - accuracy: 0.8349 - loss: 0.4344 - val_accuracy: 0.5625 - val_loss: 1.4200
Epoch 37/55
21/21 - 0s - 5ms/step - accuracy: 0.8410 - loss: 0.4301 - val_accuracy: 0.5625 - val_loss: 1.3950
Epoch 38/55
21/21 - 0s - 4ms/step - accuracy: 0.8256 - loss: 0.4345 - val_accuracy: 0.5312 - val_loss: 1.5121
Epoch 39/55
21/21 - 0s - 4ms/step - accuracy: 0.8549 - loss: 0.4144 - val_accuracy: 0.5469 - val_loss: 1.3401
Epoch 40/55
21/21 - 0s - 5ms/step - accuracy: 0.8272 - loss: 0.4411 - val_accuracy: 0.5625 - val_loss: 1.3560
Epoch 41/55
21/21 - 0s - 5ms/step - accuracy: 0.8395 - loss: 0.4027 - val_accuracy: 0.6094 - val_loss: 1.2862
Epoch 42/55
21/21 - 0s - 4ms/step - accuracy: 0.8426 - loss: 0.4296 - val_accuracy: 0.6250 - val_loss: 1.4104
Epoch 43/55
21/21 - 0s - 4ms/step - accuracy: 0.8410 - loss: 0.4294 - val_accuracy: 0.6250 - val_loss: 1.3508
Epoch 44/55
21/21 - 0s - 4ms/step - accuracy: 0.8534 - loss: 0.3997 - val_accuracy: 0.5938 - val_loss: 1.3688
Epoch 45/55
21/21 - 0s - 4ms/step - accuracy: 0.8642 - loss: 0.4031 - val_accuracy: 0.6250 - val_loss: 1.3021
Epoch 46/55
21/21 - 0s - 5ms/step - accuracy: 0.8395 - loss: 0.4113 - val_accuracy: 0.5000 - val_loss: 1.5081
Epoch 47/55
21/21 - 0s - 5ms/step - accuracy: 0.8503 - loss: 0.4070 - val_accuracy: 0.5938 - val_loss: 1.3440
Epoch 48/55
21/21 - 0s - 4ms/step - accuracy: 0.8565 - loss: 0.3772 - val_accuracy: 0.5625 - val_loss: 1.3175
Epoch 49/55
21/21 - 0s - 4ms/step - accuracy: 0.8426 - loss: 0.4264 - val_accuracy: 0.5781 - val_loss: 1.4258
Epoch 50/55
21/21 - 0s - 4ms/step - accuracy: 0.8426 - loss: 0.4380 - val_accuracy: 0.5312 - val_loss: 1.4044
Epoch 51/55
21/21 - 0s - 5ms/step - accuracy: 0.8302 - loss: 0.4335 - val_accuracy: 0.5938 - val_loss: 1.3202
Epoch 52/55
21/21 - 0s - 5ms/step - accuracy: 0.8194 - loss: 0.4365 - val_accuracy: 0.5156 - val_loss: 1.4240
Epoch 53/55
21/21 - 0s - 4ms/step - accuracy: 0.8642 - loss: 0.4053 - val_accuracy: 0.5000 - val_loss: 1.4739
Epoch 54/55
21/21 - 0s - 4ms/step - accuracy: 0.8441 - loss: 0.4029 - val_accuracy: 0.6094 - val_loss: 1.5367
Epoch 55/55
21/21 - 0s - 4ms/step - accuracy: 0.8410 - loss: 0.4212 - val_accuracy: 0.5938 - val_loss: 1.3132
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 18ms/step
Precision: 0.6895
Recall: 0.5938
F1-Score: 0.6299

plot_confusion_matrices_nn(tuned_model_gl_res, 
                        X_train_gl_res, np.argmax(y_train_gl_res, axis=1), 
                        X_val_gl, np.argmax(y_val, axis=1), 
                        X_test_gl, np.argmax(y_test, axis=1))

21/21 ━━━━━━━━━━━━━━━━━━━━ 0s 4ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 12ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 14ms/step

#Calculating different metrics on tuned_gl data
tuned_metrics_gl_res = model_performance_classification_sklearn_nn(tuned_model_gl_res, 
                        X_train_gl_res, np.argmax(y_train_gl_res, axis=1), 
                        X_val_gl, np.argmax(y_val, axis=1), 
                        X_test_gl, np.argmax(y_test, axis=1))
print("Model performance:\n")
tuned_metrics_gl_res

21/21 ━━━━━━━━━━━━━━━━━━━━ 0s 2ms/step 
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 13ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 15ms/step
Model performance:

# Train and evaluate the base model
print("Evaluating base model:")
base_model_st_res, base_precision_st_res, base_recall_st_res, base_f1score_st_res = train_and_evaluate_model(X_train_st_res, X_val_st, y_train_st_res, y_val, use_tuned_model=False)

Evaluating base model:
X_train shape: (648, 384)
y_train shape: (648, 3)

📌 Base Model Summary:

Model: "sequential_10"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Layer (type)                         ┃ Output Shape                ┃         Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ dense_30 (Dense)                     │ (None, 32)                  │          12,320 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_20               │ (None, 32)                  │             128 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_20 (Dropout)                 │ (None, 32)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_31 (Dense)                     │ (None, 16)                  │             528 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_21               │ (None, 16)                  │              64 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_21 (Dropout)                 │ (None, 16)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_32 (Dense)                     │ (None, 3)                   │              51 │
└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘

 Total params: 13,091 (51.14 KB)

 Trainable params: 12,995 (50.76 KB)

 Non-trainable params: 96 (384.00 B)

Epoch 1/55
21/21 - 2s - 83ms/step - accuracy: 0.4182 - loss: 1.4588 - val_accuracy: 0.1406 - val_loss: 1.1361
Epoch 2/55
21/21 - 0s - 4ms/step - accuracy: 0.5062 - loss: 1.2554 - val_accuracy: 0.1406 - val_loss: 1.1568
Epoch 3/55
21/21 - 0s - 4ms/step - accuracy: 0.5509 - loss: 1.0799 - val_accuracy: 0.1406 - val_loss: 1.1840
Epoch 4/55
21/21 - 0s - 4ms/step - accuracy: 0.5386 - loss: 1.0472 - val_accuracy: 0.1406 - val_loss: 1.1926
Epoch 5/55
21/21 - 0s - 4ms/step - accuracy: 0.6219 - loss: 0.9029 - val_accuracy: 0.1406 - val_loss: 1.1903
Epoch 6/55
21/21 - 0s - 4ms/step - accuracy: 0.6281 - loss: 0.8389 - val_accuracy: 0.1406 - val_loss: 1.1863
Epoch 7/55
21/21 - 0s - 4ms/step - accuracy: 0.6636 - loss: 0.8192 - val_accuracy: 0.1406 - val_loss: 1.1931
Epoch 8/55
21/21 - 0s - 4ms/step - accuracy: 0.6605 - loss: 0.7968 - val_accuracy: 0.1406 - val_loss: 1.1975
Epoch 9/55
21/21 - 0s - 4ms/step - accuracy: 0.7099 - loss: 0.6877 - val_accuracy: 0.2500 - val_loss: 1.1568
Epoch 10/55
21/21 - 0s - 4ms/step - accuracy: 0.7160 - loss: 0.6514 - val_accuracy: 0.2969 - val_loss: 1.1204
Epoch 11/55
21/21 - 0s - 4ms/step - accuracy: 0.7284 - loss: 0.6414 - val_accuracy: 0.3906 - val_loss: 1.0764
Epoch 12/55
21/21 - 0s - 4ms/step - accuracy: 0.7346 - loss: 0.6362 - val_accuracy: 0.4219 - val_loss: 1.0580
Epoch 13/55
21/21 - 0s - 4ms/step - accuracy: 0.7407 - loss: 0.5959 - val_accuracy: 0.4531 - val_loss: 1.0360
Epoch 14/55
21/21 - 0s - 4ms/step - accuracy: 0.7762 - loss: 0.5700 - val_accuracy: 0.4844 - val_loss: 1.0344
Epoch 15/55
21/21 - 0s - 4ms/step - accuracy: 0.7593 - loss: 0.5769 - val_accuracy: 0.5156 - val_loss: 1.0031
Epoch 16/55
21/21 - 0s - 4ms/step - accuracy: 0.7593 - loss: 0.5953 - val_accuracy: 0.4844 - val_loss: 0.9394
Epoch 17/55
21/21 - 0s - 4ms/step - accuracy: 0.7948 - loss: 0.5343 - val_accuracy: 0.5156 - val_loss: 0.9271
Epoch 18/55
21/21 - 0s - 4ms/step - accuracy: 0.7932 - loss: 0.5166 - val_accuracy: 0.5312 - val_loss: 0.8930
Epoch 19/55
21/21 - 0s - 4ms/step - accuracy: 0.8056 - loss: 0.4815 - val_accuracy: 0.5469 - val_loss: 0.8553
Epoch 20/55
21/21 - 0s - 4ms/step - accuracy: 0.8380 - loss: 0.4496 - val_accuracy: 0.6250 - val_loss: 0.8433
Epoch 21/55
21/21 - 0s - 4ms/step - accuracy: 0.8410 - loss: 0.4605 - val_accuracy: 0.6406 - val_loss: 0.7891
Epoch 22/55
21/21 - 0s - 4ms/step - accuracy: 0.8488 - loss: 0.4477 - val_accuracy: 0.6719 - val_loss: 0.7652
Epoch 23/55
21/21 - 0s - 4ms/step - accuracy: 0.8287 - loss: 0.4818 - val_accuracy: 0.6875 - val_loss: 0.7658
Epoch 24/55
21/21 - 0s - 4ms/step - accuracy: 0.8457 - loss: 0.4299 - val_accuracy: 0.7031 - val_loss: 0.7325
Epoch 25/55
21/21 - 0s - 4ms/step - accuracy: 0.8441 - loss: 0.4353 - val_accuracy: 0.6562 - val_loss: 0.7228
Epoch 26/55
21/21 - 0s - 4ms/step - accuracy: 0.8410 - loss: 0.3925 - val_accuracy: 0.6719 - val_loss: 0.7096
Epoch 27/55
21/21 - 0s - 4ms/step - accuracy: 0.8426 - loss: 0.4232 - val_accuracy: 0.6875 - val_loss: 0.6885
Epoch 28/55
21/21 - 0s - 5ms/step - accuracy: 0.8873 - loss: 0.3486 - val_accuracy: 0.6875 - val_loss: 0.6958
Epoch 29/55
21/21 - 0s - 5ms/step - accuracy: 0.8657 - loss: 0.3624 - val_accuracy: 0.7031 - val_loss: 0.6695
Epoch 30/55
21/21 - 0s - 5ms/step - accuracy: 0.8920 - loss: 0.3142 - val_accuracy: 0.6719 - val_loss: 0.6656
Epoch 31/55
21/21 - 0s - 4ms/step - accuracy: 0.8673 - loss: 0.3710 - val_accuracy: 0.6875 - val_loss: 0.6670
Epoch 32/55
21/21 - 0s - 4ms/step - accuracy: 0.8843 - loss: 0.3403 - val_accuracy: 0.7031 - val_loss: 0.6836
Epoch 33/55
21/21 - 0s - 4ms/step - accuracy: 0.8904 - loss: 0.3441 - val_accuracy: 0.7031 - val_loss: 0.7182
Epoch 34/55
21/21 - 0s - 4ms/step - accuracy: 0.8611 - loss: 0.3628 - val_accuracy: 0.7031 - val_loss: 0.7270
Epoch 35/55
21/21 - 0s - 4ms/step - accuracy: 0.8997 - loss: 0.3170 - val_accuracy: 0.6875 - val_loss: 0.7456
Epoch 36/55
21/21 - 0s - 4ms/step - accuracy: 0.8951 - loss: 0.3226 - val_accuracy: 0.7188 - val_loss: 0.7071
Epoch 37/55
21/21 - 0s - 4ms/step - accuracy: 0.9012 - loss: 0.3054 - val_accuracy: 0.7031 - val_loss: 0.7260
Epoch 38/55
21/21 - 0s - 4ms/step - accuracy: 0.9012 - loss: 0.3044 - val_accuracy: 0.7188 - val_loss: 0.7320
Epoch 39/55
21/21 - 0s - 4ms/step - accuracy: 0.9090 - loss: 0.3039 - val_accuracy: 0.7031 - val_loss: 0.7280
Epoch 40/55
21/21 - 0s - 4ms/step - accuracy: 0.9090 - loss: 0.2750 - val_accuracy: 0.7031 - val_loss: 0.7604
Epoch 41/55
21/21 - 0s - 4ms/step - accuracy: 0.8827 - loss: 0.3119 - val_accuracy: 0.7031 - val_loss: 0.7488
Epoch 42/55
21/21 - 0s - 4ms/step - accuracy: 0.8920 - loss: 0.3040 - val_accuracy: 0.7031 - val_loss: 0.7898
Epoch 43/55
21/21 - 0s - 4ms/step - accuracy: 0.8997 - loss: 0.3002 - val_accuracy: 0.7188 - val_loss: 0.7668
Epoch 44/55
21/21 - 0s - 4ms/step - accuracy: 0.9043 - loss: 0.2658 - val_accuracy: 0.7344 - val_loss: 0.7778
Epoch 45/55
21/21 - 0s - 4ms/step - accuracy: 0.9090 - loss: 0.2779 - val_accuracy: 0.7344 - val_loss: 0.7836
Epoch 46/55
21/21 - 0s - 4ms/step - accuracy: 0.9398 - loss: 0.2206 - val_accuracy: 0.7344 - val_loss: 0.8073
Epoch 47/55
21/21 - 0s - 5ms/step - accuracy: 0.8904 - loss: 0.2646 - val_accuracy: 0.7344 - val_loss: 0.8190
Epoch 48/55
21/21 - 0s - 4ms/step - accuracy: 0.9151 - loss: 0.2520 - val_accuracy: 0.7500 - val_loss: 0.8439
Epoch 49/55
21/21 - 0s - 5ms/step - accuracy: 0.8858 - loss: 0.3136 - val_accuracy: 0.7188 - val_loss: 0.8049
Epoch 50/55
21/21 - 0s - 4ms/step - accuracy: 0.9198 - loss: 0.2500 - val_accuracy: 0.7188 - val_loss: 0.8567
Epoch 51/55
21/21 - 0s - 4ms/step - accuracy: 0.9105 - loss: 0.2467 - val_accuracy: 0.7500 - val_loss: 0.8088
Epoch 52/55
21/21 - 0s - 4ms/step - accuracy: 0.9290 - loss: 0.2192 - val_accuracy: 0.7500 - val_loss: 0.8310
Epoch 53/55
21/21 - 0s - 5ms/step - accuracy: 0.9228 - loss: 0.2398 - val_accuracy: 0.7344 - val_loss: 0.8369
Epoch 54/55
21/21 - 0s - 5ms/step - accuracy: 0.9414 - loss: 0.1977 - val_accuracy: 0.7188 - val_loss: 0.8565
Epoch 55/55
21/21 - 0s - 4ms/step - accuracy: 0.9259 - loss: 0.2464 - val_accuracy: 0.6875 - val_loss: 0.9038
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 13ms/step
Precision: 0.7082
Recall: 0.6875
F1-Score: 0.6955

plot_confusion_matrices_nn(base_model_st_res, 
                        X_train_st_res, np.argmax(y_train_st_res, axis=1), 
                        X_val_st, np.argmax(y_val, axis=1), 
                        X_test_st, np.argmax(y_test, axis=1))

21/21 ━━━━━━━━━━━━━━━━━━━━ 0s 4ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 14ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 12ms/step

#Calculating different metrics on base_st data
base_metrics_st_res = model_performance_classification_sklearn_nn(base_model_st_res, 
                        X_train_st_res, np.argmax(y_train_st_res, axis=1), 
                        X_val_st, np.argmax(y_val, axis=1), 
                        X_test_st, np.argmax(y_test, axis=1))
print("Model performance:\n")
base_metrics_st_res

21/21 ━━━━━━━━━━━━━━━━━━━━ 0s 1ms/step 
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 11ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 13ms/step
Model performance:

# Train and evaluate the tuned model
print("\nEvaluating tuned model:")
tuned_model_st_res, tuned_precision_st_res, tuned_recall_st_res, tuned_f1score_st_res = train_and_evaluate_model(X_train_st_res, X_val_st, y_train_st_res, y_val, use_tuned_model=True)

Evaluating tuned model:
X_train shape: (648, 384)
y_train shape: (648, 3)
Reloading Tuner from tuner_results\nn_tuning\tuner0.json

📌 Best Tuned Model Summary:

Model: "sequential_11"

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Layer (type)                         ┃ Output Shape                ┃         Param # ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ dense_33 (Dense)                     │ (None, 64)                  │          24,640 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_22               │ (None, 64)                  │             256 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_22 (Dropout)                 │ (None, 64)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_34 (Dense)                     │ (None, 64)                  │           4,160 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ batch_normalization_23               │ (None, 64)                  │             256 │
│ (BatchNormalization)                 │                             │                 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dropout_23 (Dropout)                 │ (None, 64)                  │               0 │
├──────────────────────────────────────┼─────────────────────────────┼─────────────────┤
│ dense_35 (Dense)                     │ (None, 3)                   │             195 │
└──────────────────────────────────────┴─────────────────────────────┴─────────────────┘

 Total params: 29,507 (115.26 KB)

 Trainable params: 29,251 (114.26 KB)

 Non-trainable params: 256 (1.00 KB)

Epoch 1/55
21/21 - 2s - 77ms/step - accuracy: 0.3904 - loss: 1.5484 - val_accuracy: 0.1562 - val_loss: 1.1523
Epoch 2/55
21/21 - 0s - 4ms/step - accuracy: 0.5062 - loss: 1.2116 - val_accuracy: 0.1094 - val_loss: 1.2339
Epoch 3/55
21/21 - 0s - 5ms/step - accuracy: 0.5617 - loss: 1.0097 - val_accuracy: 0.1406 - val_loss: 1.3035
Epoch 4/55
21/21 - 0s - 5ms/step - accuracy: 0.6281 - loss: 0.8798 - val_accuracy: 0.1719 - val_loss: 1.3418
Epoch 5/55
21/21 - 0s - 4ms/step - accuracy: 0.6929 - loss: 0.7870 - val_accuracy: 0.1875 - val_loss: 1.4093
Epoch 6/55
21/21 - 0s - 4ms/step - accuracy: 0.7006 - loss: 0.7154 - val_accuracy: 0.1562 - val_loss: 1.4074
Epoch 7/55
21/21 - 0s - 4ms/step - accuracy: 0.7593 - loss: 0.6215 - val_accuracy: 0.1094 - val_loss: 1.4725
Epoch 8/55
21/21 - 0s - 4ms/step - accuracy: 0.7639 - loss: 0.5806 - val_accuracy: 0.1406 - val_loss: 1.4778
Epoch 9/55
21/21 - 0s - 5ms/step - accuracy: 0.7809 - loss: 0.5502 - val_accuracy: 0.1406 - val_loss: 1.4568
Epoch 10/55
21/21 - 0s - 4ms/step - accuracy: 0.8210 - loss: 0.4975 - val_accuracy: 0.2031 - val_loss: 1.4567
Epoch 11/55
21/21 - 0s - 5ms/step - accuracy: 0.8349 - loss: 0.4510 - val_accuracy: 0.2031 - val_loss: 1.4744
Epoch 12/55
21/21 - 0s - 4ms/step - accuracy: 0.8503 - loss: 0.4141 - val_accuracy: 0.2031 - val_loss: 1.4532
Epoch 13/55
21/21 - 0s - 4ms/step - accuracy: 0.8596 - loss: 0.3787 - val_accuracy: 0.2344 - val_loss: 1.4573
Epoch 14/55
21/21 - 0s - 4ms/step - accuracy: 0.8596 - loss: 0.3968 - val_accuracy: 0.2656 - val_loss: 1.4018
Epoch 15/55
21/21 - 0s - 4ms/step - accuracy: 0.8750 - loss: 0.3558 - val_accuracy: 0.3125 - val_loss: 1.3125
Epoch 16/55
21/21 - 0s - 4ms/step - accuracy: 0.8997 - loss: 0.2951 - val_accuracy: 0.3281 - val_loss: 1.2859
Epoch 17/55
21/21 - 0s - 4ms/step - accuracy: 0.8981 - loss: 0.2953 - val_accuracy: 0.3750 - val_loss: 1.2463
Epoch 18/55
21/21 - 0s - 4ms/step - accuracy: 0.8889 - loss: 0.3129 - val_accuracy: 0.4844 - val_loss: 1.1306
Epoch 19/55
21/21 - 0s - 5ms/step - accuracy: 0.9043 - loss: 0.2687 - val_accuracy: 0.4531 - val_loss: 1.1535
Epoch 20/55
21/21 - 0s - 5ms/step - accuracy: 0.8920 - loss: 0.2974 - val_accuracy: 0.4688 - val_loss: 1.1148
Epoch 21/55
21/21 - 0s - 4ms/step - accuracy: 0.9213 - loss: 0.2330 - val_accuracy: 0.4844 - val_loss: 1.0051
Epoch 22/55
21/21 - 0s - 4ms/step - accuracy: 0.9136 - loss: 0.2447 - val_accuracy: 0.5156 - val_loss: 1.0035
Epoch 23/55
21/21 - 0s - 5ms/step - accuracy: 0.9321 - loss: 0.2245 - val_accuracy: 0.5312 - val_loss: 0.9991
Epoch 24/55
21/21 - 0s - 4ms/step - accuracy: 0.9228 - loss: 0.2163 - val_accuracy: 0.5312 - val_loss: 0.9485
Epoch 25/55
21/21 - 0s - 5ms/step - accuracy: 0.9244 - loss: 0.2265 - val_accuracy: 0.5781 - val_loss: 0.8656
Epoch 26/55
21/21 - 0s - 4ms/step - accuracy: 0.9552 - loss: 0.1497 - val_accuracy: 0.5938 - val_loss: 0.8753
Epoch 27/55
21/21 - 0s - 4ms/step - accuracy: 0.9460 - loss: 0.1634 - val_accuracy: 0.6250 - val_loss: 0.8253
Epoch 28/55
21/21 - 0s - 4ms/step - accuracy: 0.9321 - loss: 0.2034 - val_accuracy: 0.6094 - val_loss: 0.8529
Epoch 29/55
21/21 - 0s - 4ms/step - accuracy: 0.9336 - loss: 0.1968 - val_accuracy: 0.6562 - val_loss: 0.8673
Epoch 30/55
21/21 - 0s - 5ms/step - accuracy: 0.9522 - loss: 0.1517 - val_accuracy: 0.6250 - val_loss: 0.8571
Epoch 31/55
21/21 - 0s - 5ms/step - accuracy: 0.9522 - loss: 0.1611 - val_accuracy: 0.6406 - val_loss: 0.8354
Epoch 32/55
21/21 - 0s - 4ms/step - accuracy: 0.9429 - loss: 0.1724 - val_accuracy: 0.6406 - val_loss: 0.8419
Epoch 33/55
21/21 - 0s - 4ms/step - accuracy: 0.9460 - loss: 0.1702 - val_accuracy: 0.7188 - val_loss: 0.7751
Epoch 34/55
21/21 - 0s - 4ms/step - accuracy: 0.9522 - loss: 0.1391 - val_accuracy: 0.6875 - val_loss: 0.8053
Epoch 35/55
21/21 - 0s - 4ms/step - accuracy: 0.9444 - loss: 0.1776 - val_accuracy: 0.6875 - val_loss: 0.9164
Epoch 36/55
21/21 - 0s - 4ms/step - accuracy: 0.9506 - loss: 0.1387 - val_accuracy: 0.6562 - val_loss: 0.9958
Epoch 37/55
21/21 - 0s - 4ms/step - accuracy: 0.9321 - loss: 0.1951 - val_accuracy: 0.6406 - val_loss: 0.9317
Epoch 38/55
21/21 - 0s - 4ms/step - accuracy: 0.9614 - loss: 0.1202 - val_accuracy: 0.6875 - val_loss: 0.8886
Epoch 39/55
21/21 - 0s - 4ms/step - accuracy: 0.9660 - loss: 0.1152 - val_accuracy: 0.6875 - val_loss: 0.9320
Epoch 40/55
21/21 - 0s - 5ms/step - accuracy: 0.9552 - loss: 0.1418 - val_accuracy: 0.6875 - val_loss: 0.9276
Epoch 41/55
21/21 - 0s - 4ms/step - accuracy: 0.9614 - loss: 0.1365 - val_accuracy: 0.6719 - val_loss: 0.9854
Epoch 42/55
21/21 - 0s - 4ms/step - accuracy: 0.9630 - loss: 0.1110 - val_accuracy: 0.6719 - val_loss: 1.0371
Epoch 43/55
21/21 - 0s - 4ms/step - accuracy: 0.9660 - loss: 0.0998 - val_accuracy: 0.6719 - val_loss: 1.0708
Epoch 44/55
21/21 - 0s - 4ms/step - accuracy: 0.9583 - loss: 0.1090 - val_accuracy: 0.6562 - val_loss: 1.0086
Epoch 45/55
21/21 - 0s - 4ms/step - accuracy: 0.9583 - loss: 0.1355 - val_accuracy: 0.6562 - val_loss: 0.9557
Epoch 46/55
21/21 - 0s - 4ms/step - accuracy: 0.9599 - loss: 0.1267 - val_accuracy: 0.6719 - val_loss: 0.9207
Epoch 47/55
21/21 - 0s - 4ms/step - accuracy: 0.9691 - loss: 0.0873 - val_accuracy: 0.6875 - val_loss: 0.9302
Epoch 48/55
21/21 - 0s - 5ms/step - accuracy: 0.9568 - loss: 0.1308 - val_accuracy: 0.6719 - val_loss: 0.9913
Epoch 49/55
21/21 - 0s - 5ms/step - accuracy: 0.9599 - loss: 0.1120 - val_accuracy: 0.6562 - val_loss: 1.0846
Epoch 50/55
21/21 - 0s - 4ms/step - accuracy: 0.9552 - loss: 0.1303 - val_accuracy: 0.6406 - val_loss: 0.9865
Epoch 51/55
21/21 - 0s - 5ms/step - accuracy: 0.9707 - loss: 0.0896 - val_accuracy: 0.6250 - val_loss: 1.0330
Epoch 52/55
21/21 - 0s - 4ms/step - accuracy: 0.9645 - loss: 0.1152 - val_accuracy: 0.7031 - val_loss: 1.0393
Epoch 53/55
21/21 - 0s - 5ms/step - accuracy: 0.9691 - loss: 0.1149 - val_accuracy: 0.7188 - val_loss: 0.9699
Epoch 54/55
21/21 - 0s - 5ms/step - accuracy: 0.9738 - loss: 0.0924 - val_accuracy: 0.7031 - val_loss: 1.0898
Epoch 55/55
21/21 - 0s - 4ms/step - accuracy: 0.9722 - loss: 0.0921 - val_accuracy: 0.7031 - val_loss: 1.0845
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 12ms/step
Precision: 0.7323
Recall: 0.7031
F1-Score: 0.7166

plot_confusion_matrices_nn(tuned_model_st_res, 
                        X_train_st_res, np.argmax(y_train_st_res, axis=1), 
                        X_val_st, np.argmax(y_val, axis=1), 
                        X_test_st, np.argmax(y_test, axis=1))

21/21 ━━━━━━━━━━━━━━━━━━━━ 0s 4ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 13ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 11ms/step

#Calculating different metrics on tuned_st data
tuned_metrics_st_res = model_performance_classification_sklearn_nn(tuned_model_st_res, 
                        X_train_st_res, np.argmax(y_train_st_res, axis=1), 
                        X_val_st, np.argmax(y_val, axis=1), 
                        X_test_st, np.argmax(y_test, axis=1))
print("Model performance:\n")
tuned_metrics_st_res

21/21 ━━━━━━━━━━━━━━━━━━━━ 0s 1ms/step 
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 14ms/step
2/2 ━━━━━━━━━━━━━━━━━━━━ 0s 12ms/step
Model performance:

def plot_confusion_matrices(trainer, train_dataset, val_dataset, test_dataset, label_list):
    
    """
    Plots confusion matrices for a trained model on training, validation, and test datasets

    Parameters:
        trainer : The trained BERT model used for predictions
        train_dataset : The dataset used for training
        val_dataset : The dataset used for validation
        test_dataset : The dataset used for testing
    """
    datasets = [train_dataset, val_dataset, test_dataset]
    titles = ['Confusion Matrix (Training)', 'Confusion Matrix (Validation)', 'Confusion Matrix (Testing)']
    
    # Define the labels for the confusion matrix
    label_list = ['High', 'Low', 'Medium']
    
    y_preds = [trainer.predict(dataset).predictions for dataset in datasets]
    y_pred_classes = [np.argmax(pred, axis=1) for pred in y_preds]
    y_true_classes = [dataset.labels for dataset in datasets]
    
    confusion_matrices = [confusion_matrix(y_true, y_pred) for y_true, y_pred in zip(y_true_classes, y_pred_classes)]
    
    fig, axes = plt.subplots(1, 3, figsize=(18, 6))
    for i, (cm, ax) in enumerate(zip(confusion_matrices, axes)):
        sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=label_list, 
                    yticklabels=label_list, cbar=False, ax=ax)
        ax.set_ylabel('Actual')
        ax.set_xlabel('Predicted')
        ax.set_title(titles[i])
    
    plt.tight_layout()
    plt.show()

def model_performance_classification(trainer, train_dataset, val_dataset, test_dataset):
    """
    Evaluates classification performance of a trained model on training, validation, and test datasets

    Parameters:
        trainer: The trained BERT model used for predictions
        train_dataset: The dataset used for training
        val_dataset: The dataset used for validation
        test_dataset: The dataset used for testing

    Returns:
        metrics_df: A DataFrame containing Accuracy, Precision, Recall, and F1-score for each dataset
    """
    datasets = {'Train': train_dataset, 'Validation': val_dataset, 'Test': test_dataset}
    metrics = {}
    reports = {}
    
    for split, dataset in datasets.items():
        y_pred = trainer.predict(dataset).predictions
        y_pred_classes = np.argmax(y_pred, axis=1)
        y_true_classes = np.array(dataset.labels)
        
        metrics[split] = [
            format(accuracy_score(y_true_classes, y_pred_classes), '.4f'),
            format(precision_score(y_true_classes, y_pred_classes, average='weighted', zero_division=0), '.4f'),
            format(recall_score(y_true_classes, y_pred_classes, average='weighted', zero_division=0), '.4f'),
            format(f1_score(y_true_classes, y_pred_classes, average='weighted'), '.4f')
        ]
        reports[split] = classification_report(y_true_classes, y_pred_classes, output_dict=True)
        print(f"Classification Report ({split}):\n", classification_report(y_true_classes, y_pred_classes))
    
    metrics_df = pd.DataFrame(metrics, index=['Accuracy', 'Precision', 'Recall', 'F1_score'])
    return metrics_df

def compute_metrics(eval_pred):
    """
    Computes accuracy, precision, recall, and F1-score for the evaluation predictions.
 
    Parameters:
        eval_pred: Tuple containing (logits, labels)
    Returns:
        Dictionary containing accuracy, precision, recall, and F1-score.
    """
    logits, labels = eval_pred
    preds = np.argmax(logits, axis=-1)  # Convert logits to predicted labels
 
    accuracy = accuracy_score(labels, preds)
    precision = precision_score(labels, preds, average="weighted")
    recall = recall_score(labels, preds, average="weighted")
    f1 = f1_score(labels, preds, average="weighted")
 
    return {
        "accuracy": accuracy,
        "precision": precision,
        "recall": recall,
        "f1": f1
    }

def train_bert_model(X_train, X_test, X_val, y_train, y_test, y_val, use_smote=False, epochs=3, batch_size=8, learning_rate=0.01):
    """
    Trains a BERT model for classification on industrial safety data

    Parameters:
        X_train, X_test, X_val: Text data for training, testing, and validation
        y_train, y_test, y_val: Corresponding labels (one-hot encoded or categorical)
        use_smote: Whether to apply SMOTE for handling class imbalance (default = False)
        epochs: Number of training epochs (default = 3)
        batch_size: Batch size for training and evaluation (default = 8)
        learning_rate: Learning rate for the optimizer (default = 0.01)

    Returns:
        trainer: The trained BERT model wrapped with Hugging Face's Trainer
        train_dataset, val_dataset, test_dataset : Processed datasets for further evaluation
    """
    # set 12 cpu's
    torch.set_num_threads(12)
    
    # Load BERT tokenizer
    tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")

    # Tokenize input data
    def tokenize_function(texts):
        return tokenizer(texts, padding=True, truncation=True, max_length=512, return_tensors="pt")

    train_encodings = tokenize_function(X_train.tolist())
    test_encodings = tokenize_function(X_test.tolist())
    val_encodings = tokenize_function(X_val.tolist())

    if use_smote:
        # Apply SMOTE for oversampling
        smote = SMOTE(random_state=42)
        input_ids_resampled, y_train_resampled = smote.fit_resample(train_encodings['input_ids'], np.argmax(y_train, axis=1))
        attention_masks_resampled = np.where(input_ids_resampled != 0, 1, 0)

        train_encodings = {
            "input_ids": torch.tensor(input_ids_resampled),
            "attention_mask": torch.tensor(attention_masks_resampled)
        }
        y_train = y_train_resampled

    # Define Dataset class
    class AccidentDataset(Dataset):
        def __init__(self, encodings, labels):
            self.encodings = encodings
            self.labels = labels if len(labels.shape) == 1 else np.argmax(labels, axis=1)

        def __len__(self):
            return len(self.labels)

        def __getitem__(self, idx):
            item = {key: val[idx] for key, val in self.encodings.items()}
            item["labels"] = torch.tensor(self.labels[idx], dtype=torch.long)
            return item

    # Create datasets
    train_dataset = AccidentDataset(train_encodings, y_train)
    test_dataset = AccidentDataset(test_encodings, np.argmax(y_test, axis=1))
    val_dataset = AccidentDataset(val_encodings, np.argmax(y_val, axis=1))

    # Load BERT model
    model = BertForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=9)

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model.to(device)
    
    # Training arguments
    training_args = TrainingArguments(
        output_dir="./results",
        evaluation_strategy="epoch",
        save_strategy="epoch",
        per_device_train_batch_size=batch_size,
        per_device_eval_batch_size=batch_size,
        num_train_epochs=epochs,
        weight_decay=0.01,
        logging_dir="./logs",
        logging_steps=10,
        learning_rate=learning_rate
    )

    # Trainer
    trainer = Trainer(
        model=model,
        args=training_args,
        train_dataset=train_dataset,
        eval_dataset=val_dataset,
        compute_metrics=compute_metrics
    )

    # Train the model
    trainer.train()

    # Evaluate the model
    results = trainer.evaluate(test_dataset)
    print("Evaluation Results:", results)

    return trainer, train_dataset, val_dataset, test_dataset

trainer, train_dataset, val_dataset, test_dataset = train_bert_model(X_train, X_val, X_test, y_train, y_val, y_test, use_smote=False, epochs=3, batch_size=8, learning_rate=0.01)

Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Evaluation Results: {'eval_loss': 0.6996376514434814, 'eval_accuracy': 0.78125, 'eval_precision': 0.6103515625, 'eval_recall': 0.78125, 'eval_f1': 0.6853070175438596, 'eval_runtime': 2.772, 'eval_samples_per_second': 23.088, 'eval_steps_per_second': 2.886, 'epoch': 3.0}

label_list = ['High', 'Low', 'Medium']
plot_confusion_matrices(trainer, train_dataset, val_dataset, test_dataset, label_list)
base_bert_metrics = model_performance_classification(trainer, train_dataset, val_dataset, test_dataset)
print("\nBERT Model Performance:\n")
base_bert_metrics

Classification Report (Train):
               precision    recall  f1-score   support

           0       0.00      0.00      0.00        29
           1       0.73      1.00      0.84       216
           2       0.00      0.00      0.00        52

    accuracy                           0.73       297
   macro avg       0.24      0.33      0.28       297
weighted avg       0.53      0.73      0.61       297

Classification Report (Validation):
               precision    recall  f1-score   support

           0       0.00      0.00      0.00         4
           1       0.78      1.00      0.88        50
           2       0.00      0.00      0.00        10

    accuracy                           0.78        64
   macro avg       0.26      0.33      0.29        64
weighted avg       0.61      0.78      0.69        64

Classification Report (Test):
               precision    recall  f1-score   support

           0       0.00      0.00      0.00         5
           1       0.78      1.00      0.88        50
           2       0.00      0.00      0.00         9

    accuracy                           0.78        64
   macro avg       0.26      0.33      0.29        64
weighted avg       0.61      0.78      0.69        64


BERT Model Performance:

trainer, train_dataset_resampled, val_dataset, test_dataset = train_bert_model(X_train, X_val, X_test, y_train, y_val, y_test, use_smote=True, epochs=3, batch_size=8, learning_rate=0.01)

Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Evaluation Results: {'eval_loss': 0.9402669668197632, 'eval_accuracy': 0.78125, 'eval_precision': 0.6103515625, 'eval_recall': 0.78125, 'eval_f1': 0.6853070175438596, 'eval_runtime': 3.8155, 'eval_samples_per_second': 16.774, 'eval_steps_per_second': 2.097, 'epoch': 3.0}

plot_confusion_matrices(trainer, train_dataset_resampled, val_dataset, test_dataset, label_list)
base_bert_metrics_res = model_performance_classification(trainer, train_dataset_resampled, val_dataset, test_dataset)
print("\nBERT Model Performance (Resampled Data):\n")
base_bert_metrics_res

Classification Report (Train):
               precision    recall  f1-score   support

           0       0.00      0.00      0.00       216
           1       0.33      1.00      0.50       216
           2       0.00      0.00      0.00       216

    accuracy                           0.33       648
   macro avg       0.11      0.33      0.17       648
weighted avg       0.11      0.33      0.17       648

Classification Report (Validation):
               precision    recall  f1-score   support

           0       0.00      0.00      0.00         4
           1       0.78      1.00      0.88        50
           2       0.00      0.00      0.00        10

    accuracy                           0.78        64
   macro avg       0.26      0.33      0.29        64
weighted avg       0.61      0.78      0.69        64

Classification Report (Test):
               precision    recall  f1-score   support

           0       0.00      0.00      0.00         5
           1       0.78      1.00      0.88        50
           2       0.00      0.00      0.00         9

    accuracy                           0.78        64
   macro avg       0.26      0.33      0.29        64
weighted avg       0.61      0.78      0.69        64


BERT Model Performance (Resampled Data):

# Hyperparameter tuning function
def objective(trial, X_train, X_test, X_val, y_train, y_test, y_val, use_smote=False):
    learning_rate = trial.suggest_loguniform("learning_rate", 2e-5, 5e-5)
    batch_size = trial.suggest_categorical("batch_size", [8, 16, 32])
    epochs = trial.suggest_int("epochs", 3, 5)

    trainer, _, _, _ = train_bert_model(X_train, X_test, X_val, y_train, y_test, y_val, use_smote=False,
                                        epochs=epochs, batch_size=batch_size, learning_rate=learning_rate)
    
    val_results = trainer.evaluate()
    return val_results["eval_loss"]

# Function to run hyperparameter tuning
def tune_bert_hyperparameters(X_train, X_test, X_val, y_train, y_test, y_val, n_trials=10, use_smote=False):
    pruner = optuna.pruners.MedianPruner(n_warmup_steps = 2)
    study = optuna.create_study(direction="minimize", pruner=pruner, sampler=optuna.samplers.TPESampler(seed=42))
    study.optimize(lambda trial: objective(trial, X_train, X_test, X_val, y_train, y_test, y_val, use_smote=False), 
                   n_trials=n_trials, timeout = 1800)
    
    print("Best hyperparameters:", study.best_params)
    return study.best_params

best_params = tune_bert_hyperparameters(X_train, X_test, X_val, y_train, y_test, y_val, use_smote=False)

[I 2025-03-02 20:38:44,322] A new study created in memory with name: no-name-10b1d841-2392-423e-8c6b-75bd42aef12b
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Evaluation Results: {'eval_loss': 0.6871282458305359, 'eval_accuracy': 0.78125, 'eval_precision': 0.6103515625, 'eval_recall': 0.78125, 'eval_f1': 0.6853070175438596, 'eval_runtime': 1.9909, 'eval_samples_per_second': 32.146, 'eval_steps_per_second': 4.018, 'epoch': 3.0}

[I 2025-03-02 20:41:50,475] Trial 0 finished with value: 0.6837172508239746 and parameters: {'learning_rate': 2.8188664052384835e-05, 'batch_size': 8, 'epochs': 3}. Best is trial 0 with value: 0.6837172508239746.
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Evaluation Results: {'eval_loss': 0.714826762676239, 'eval_accuracy': 0.78125, 'eval_precision': 0.6103515625, 'eval_recall': 0.78125, 'eval_f1': 0.6853070175438596, 'eval_runtime': 1.911, 'eval_samples_per_second': 33.49, 'eval_steps_per_second': 2.093, 'epoch': 5.0}

[I 2025-03-02 20:46:26,200] Trial 1 finished with value: 0.6742880344390869 and parameters: {'learning_rate': 2.3073126994560993e-05, 'batch_size': 16, 'epochs': 5}. Best is trial 1 with value: 0.6742880344390869.
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Evaluation Results: {'eval_loss': 0.705443263053894, 'eval_accuracy': 0.78125, 'eval_precision': 0.6103515625, 'eval_recall': 0.78125, 'eval_f1': 0.6853070175438596, 'eval_runtime': 2.0002, 'eval_samples_per_second': 31.997, 'eval_steps_per_second': 4.0, 'epoch': 3.0}

[I 2025-03-02 20:49:26,889] Trial 2 finished with value: 0.6968289017677307 and parameters: {'learning_rate': 2.0380807616360226e-05, 'batch_size': 8, 'epochs': 3}. Best is trial 1 with value: 0.6742880344390869.
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Evaluation Results: {'eval_loss': 0.7581973075866699, 'eval_accuracy': 0.78125, 'eval_precision': 0.6103515625, 'eval_recall': 0.78125, 'eval_f1': 0.6853070175438596, 'eval_runtime': 1.9156, 'eval_samples_per_second': 33.41, 'eval_steps_per_second': 2.088, 'epoch': 3.0}

[I 2025-03-02 20:52:14,817] Trial 3 finished with value: 0.7380400896072388 and parameters: {'learning_rate': 2.3659959010655903e-05, 'batch_size': 16, 'epochs': 3}. Best is trial 1 with value: 0.6742880344390869.
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Evaluation Results: {'eval_loss': 0.7548090815544128, 'eval_accuracy': 0.78125, 'eval_precision': 0.6103515625, 'eval_recall': 0.78125, 'eval_f1': 0.6853070175438596, 'eval_runtime': 1.9251, 'eval_samples_per_second': 33.245, 'eval_steps_per_second': 1.039, 'epoch': 4.0}

trainer, train_dataset, val_dataset, test_dataset = train_bert_model(
    X_train, X_test, X_val, y_train, y_test, y_val, use_smote=False,
    batch_size=best_params["batch_size"],
    learning_rate=best_params["learning_rate"],
    epochs=best_params["epochs"]
)

Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Evaluation Results: {'eval_loss': 0.7007152438163757, 'eval_accuracy': 0.78125, 'eval_precision': 0.6103515625, 'eval_recall': 0.78125, 'eval_f1': 0.6853070175438596, 'eval_runtime': 1.9036, 'eval_samples_per_second': 33.62, 'eval_steps_per_second': 2.101, 'epoch': 5.0}

plot_confusion_matrices(trainer, train_dataset, val_dataset, test_dataset, label_list)
tuned_bert_metrics = model_performance_classification(trainer, train_dataset, val_dataset, test_dataset)
print("\nBERT Model Performance (Tuned):\n")
tuned_bert_metrics

Classification Report (Train):
               precision    recall  f1-score   support

           0       0.00      0.00      0.00        29
           1       0.73      1.00      0.84       216
           2       0.00      0.00      0.00        52

    accuracy                           0.73       297
   macro avg       0.24      0.33      0.28       297
weighted avg       0.53      0.73      0.61       297

Classification Report (Validation):
               precision    recall  f1-score   support

           0       0.00      0.00      0.00         5
           1       0.78      1.00      0.88        50
           2       0.00      0.00      0.00         9

    accuracy                           0.78        64
   macro avg       0.26      0.33      0.29        64
weighted avg       0.61      0.78      0.69        64

Classification Report (Test):
               precision    recall  f1-score   support

           0       0.00      0.00      0.00         4
           1       0.78      1.00      0.88        50
           2       0.00      0.00      0.00        10

    accuracy                           0.78        64
   macro avg       0.26      0.33      0.29        64
weighted avg       0.61      0.78      0.69        64


BERT Model Performance (Tuned):

best_params = tune_bert_hyperparameters(X_train, X_test, X_val, y_train, y_test, y_val, use_smote=True)

[I 2025-03-02 21:14:44,881] A new study created in memory with name: no-name-a452ee1a-cf6a-401c-bc48-c6129a154aa7
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Evaluation Results: {'eval_loss': 0.717431366443634, 'eval_accuracy': 0.78125, 'eval_precision': 0.6103515625, 'eval_recall': 0.78125, 'eval_f1': 0.6853070175438596, 'eval_runtime': 1.9509, 'eval_samples_per_second': 32.805, 'eval_steps_per_second': 4.101, 'epoch': 3.0}

[I 2025-03-02 21:17:40,146] Trial 0 finished with value: 0.709922194480896 and parameters: {'learning_rate': 2.8188664052384835e-05, 'batch_size': 8, 'epochs': 3}. Best is trial 0 with value: 0.709922194480896.
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Evaluation Results: {'eval_loss': 0.714826762676239, 'eval_accuracy': 0.78125, 'eval_precision': 0.6103515625, 'eval_recall': 0.78125, 'eval_f1': 0.6853070175438596, 'eval_runtime': 1.8998, 'eval_samples_per_second': 33.687, 'eval_steps_per_second': 2.105, 'epoch': 5.0}

[I 2025-03-02 21:22:16,366] Trial 1 finished with value: 0.6742880344390869 and parameters: {'learning_rate': 2.3073126994560993e-05, 'batch_size': 16, 'epochs': 5}. Best is trial 1 with value: 0.6742880344390869.
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Evaluation Results: {'eval_loss': 0.705443263053894, 'eval_accuracy': 0.78125, 'eval_precision': 0.6103515625, 'eval_recall': 0.78125, 'eval_f1': 0.6853070175438596, 'eval_runtime': 1.9607, 'eval_samples_per_second': 32.641, 'eval_steps_per_second': 4.08, 'epoch': 3.0}

[I 2025-03-02 21:25:12,858] Trial 2 finished with value: 0.6968289017677307 and parameters: {'learning_rate': 2.0380807616360226e-05, 'batch_size': 8, 'epochs': 3}. Best is trial 1 with value: 0.6742880344390869.
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Evaluation Results: {'eval_loss': 0.7581973075866699, 'eval_accuracy': 0.78125, 'eval_precision': 0.6103515625, 'eval_recall': 0.78125, 'eval_f1': 0.6853070175438596, 'eval_runtime': 1.9017, 'eval_samples_per_second': 33.654, 'eval_steps_per_second': 2.103, 'epoch': 3.0}

[I 2025-03-02 21:28:01,963] Trial 3 finished with value: 0.7380400896072388 and parameters: {'learning_rate': 2.3659959010655903e-05, 'batch_size': 16, 'epochs': 3}. Best is trial 1 with value: 0.6742880344390869.
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Evaluation Results: {'eval_loss': 0.7548090815544128, 'eval_accuracy': 0.78125, 'eval_precision': 0.6103515625, 'eval_recall': 0.78125, 'eval_f1': 0.6853070175438596, 'eval_runtime': 1.8971, 'eval_samples_per_second': 33.735, 'eval_steps_per_second': 1.054, 'epoch': 4.0}

trainer, train_dataset, val_dataset, test_dataset = train_bert_model(
    X_train, X_test, X_val, y_train, y_test, y_val, use_smote=True,
    batch_size=best_params["batch_size"],
    learning_rate=best_params["learning_rate"],
    epochs=best_params["epochs"]
)

Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.

Evaluation Results: {'eval_loss': 0.7524386644363403, 'eval_accuracy': 0.765625, 'eval_precision': 0.6668374316939891, 'eval_recall': 0.765625, 'eval_f1': 0.6997141372141372, 'eval_runtime': 1.9766, 'eval_samples_per_second': 32.378, 'eval_steps_per_second': 2.024, 'epoch': 5.0}

plot_confusion_matrices(trainer, train_dataset_resampled, val_dataset, test_dataset, label_list)
tuned_bert_metrics_res = model_performance_classification(trainer, train_dataset_resampled, val_dataset, test_dataset)
print("\nBERT Model Performance (Resampled Data - Tuned):\n")
tuned_bert_metrics_res

Classification Report (Train):
               precision    recall  f1-score   support

           0       0.73      0.89      0.80       216
           1       0.90      1.00      0.95       216
           2       0.89      0.60      0.71       216

    accuracy                           0.83       648
   macro avg       0.84      0.83      0.82       648
weighted avg       0.84      0.83      0.82       648

Classification Report (Validation):
               precision    recall  f1-score   support

           0       0.00      0.00      0.00         5
           1       0.81      0.96      0.88        50
           2       0.20      0.11      0.14         9

    accuracy                           0.77        64
   macro avg       0.34      0.36      0.34        64
weighted avg       0.66      0.77      0.71        64

Classification Report (Test):
               precision    recall  f1-score   support

           0       0.00      0.00      0.00         4
           1       0.79      0.96      0.86        50
           2       0.33      0.10      0.15        10

    accuracy                           0.77        64
   macro avg       0.37      0.35      0.34        64
weighted avg       0.67      0.77      0.70        64


BERT Model Performance (Resampled Data - Tuned):

# Assuming datasets_list contains the data as DataFrames
datasets_list = [base_metrics_wv, tuned_metrics_wv, base_metrics_gl, tuned_metrics_gl,
                 base_metrics_st, tuned_metrics_st, base_bert_metrics, tuned_bert_metrics, base_metrics_wv_res,
                 tuned_metrics_wv_res, base_metrics_gl_res, tuned_metrics_gl_res, base_metrics_st_res,
                 tuned_metrics_st_res, base_bert_metrics_res, tuned_bert_metrics_res]

names = ["NN (Word2Vec) Base", "NN (Word2Vec) Tuned", "NN (GloVe) Base", "NN (GloVe) Tuned", "NN (Sentence Transformers) Base",
         "NN (Sentence Transformers) Tuned", "BERT Base", "BERT Tuned", "NN (Word2Vec Resampled) Base", 
         "NN (Word2Vec Resampled) Tuned", "NN (GloVe Resampled) Base", "NN (GloVe Resampled) Tuned", 
         "NN (Sentence Transformers Resampled) Base", "NN (Sentence Transformers Resampled) Tuned", 
         "BERT (Resampled) Base", "BERT (Resampled) Tuned"]

# Create a list to store formatted rows
formatted_rows = []

for df, model_name in zip(datasets_list, names):
    model_parts = model_name.split(" ")
    
    model_type = model_parts[0]  # NN or BERT
    model_variant = " ".join(model_parts[1:])  # Handle multi-word model names correctly

    df = df.T  # Transpose to make Train, Validation, Test rows and metrics columns

    formatted_rows.append([
        model_type,  # Model Type (NN or BERT)
        model_variant,  # Full Variant Name (e.g., "Word2Vec Resampled")
        df.iloc[0, 0], df.iloc[0, 1], df.iloc[0, 2], df.iloc[0, 3],  # Train Metrics
        df.iloc[1, 0], df.iloc[1, 1], df.iloc[1, 2], df.iloc[1, 3],  # Validation Metrics
        df.iloc[2, 0], df.iloc[2, 1], df.iloc[2, 2], df.iloc[2, 3]   # Test Metrics
    ])
    
# Define columns
columns = ["Model", "Type", "Train Accuracy", "Train Recall", "Train Precision", "Train F1 Score",
           "Val Accuracy", "Val Recall", "Val Precision", "Val F1 Score",
           "Test Accuracy", "Test Recall", "Test Precision", "Test F1 Score"]

# Create DataFrame
formatted_df = pd.DataFrame(formatted_rows, columns=columns)

# Splitting into two sections
df_original = formatted_df.iloc[:8]
df_resampled = formatted_df.iloc[8:]

# Display the data
print("\nOriginal Dataset -")
display(df_original)

print("\nResampled Dataset -")
display(df_resampled)

Original Dataset -

Resampled Dataset -

# from the above table we can observe that bert tuned resampled model has given good performance
# Open a file named 'best_model.pkl' in write-binary mode ('wb')
with open('best_model.pkl', 'wb') as file:
    # Save the model object to the file using pickle
    pickle.dump(trainer, file)

nn_best_model = tuned_model_st_res.fit(X_train_st_res, y_train_st_res, epochs=55, batch_size=32, validation_data=(X_val_st, y_val), verbose=2)

def plot_training_curves(trainer, best_model):
    """
    Plots training and validation loss/accuracy curves for both BERT and Neural Network models separately.
 
    Parameters:
        trainer: Hugging Face Trainer object containing training history for BERT
        best_model: Keras model object containing training history for Neural Network
        train_accuracy: Manually computed training accuracy for BERT
        val_accuracy: Manually computed validation accuracy for BERT
    """
    # Extract BERT training history
    history_bert = trainer.state.log_history
    epochs_bert = [i + 1 for i in range(len([entry for entry in history_bert if 'eval_loss' in entry]))]
    train_loss_bert = [entry['loss'] for entry in history_bert if 'loss' in entry]
    val_loss_bert = [entry['eval_loss'] for entry in history_bert if 'eval_loss' in entry]
 
    # Extract training accuracy for BERT (if available)
    train_accuracy_bert = [entry['eval_accuracy'] for entry in history_bert if 'eval_accuracy' in entry]
    val_accuracy_bert = train_accuracy_bert  # Since eval_accuracy is validation accuracy
 
    # Extract Neural Network training history
    history_nn = best_model.history
    epochs_nn = np.arange(1, len(history_nn['loss']) + 1)
 
    # Define metrics dictionary
    metrics = {
        "BERT": {
            "Training Loss": (train_loss_bert[:len(epochs_bert)], 'b', '-'),
            "Validation Loss": (val_loss_bert, 'orange', '--'),
            "Training Accuracy": (train_accuracy_bert, 'g', '-'),
            "Validation Accuracy": (val_accuracy_bert, 'r', '-')
        },
        "Neural Network": {
            "Training Loss": (history_nn['loss'], 'b', '-'),
            "Validation Loss": (history_nn['val_loss'], 'orange', '--'),
            "Training Accuracy": (history_nn['accuracy'], 'g', '-'),
            "Validation Accuracy": (history_nn['val_accuracy'], 'r', '-')
        }
    }
 
    # Titles and labels
    plot_titles = ["Training Loss vs. Epochs", "Training Accuracy vs. Epochs",
                   "Validation Loss vs. Epochs", "Validation Accuracy vs. Epochs"]
 
    y_labels = ["Training Loss", "Training Accuracy", "Validation Loss", "Validation Accuracy"]
 
    # Loop through models (BERT and NN)
    for model_name, data in metrics.items():
        fig, axes = plt.subplots(2, 2, figsize=(12, 9))
        fig.suptitle(f"{model_name} Training Progress", fontsize=16)
 
        # Loop through 4 plots per model
        for i, (title, ylabel) in enumerate(zip(plot_titles, y_labels)):
            ax = axes[i // 2, i % 2]
            ax.set_title(title)
            ax.set_xlabel("Epochs")
            ax.set_ylabel(ylabel)
 
            # Select correct epochs (BERT vs NN)
            epochs = epochs_bert if "BERT" in model_name else epochs_nn
 
            # Plot respective metrics
            for key, (values, color, linestyle) in data.items():
                if ylabel in key:  # Match accuracy/loss labels
                    ax.plot(epochs, values, color=color, linestyle=linestyle, label=key)
 
            ax.legend()
            ax.grid(True)
 
        plt.tight_layout(rect=[0, 0, 1, 0.96])
        plt.show()

# Calling the function with our trained models
plot_training_curves(trainer, nn_best_model)

	Description	Cleaned_Description
0	While removing the drill rod of the Jumbo 08 f...	while removing the drill rod of the jumbo for ...
1	During the activation of a sodium sulphide pum...	during the activation of a sodium sulphide pum...
2	In the sub-station MILPO located at level +170...	in the substation milpo located at level when ...
3	Being 9:45 am. approximately in the Nv. 1880 C...	being am approximately in the nv cx ob the per...
4	Approximately at 11:45 a.m. in circumstances t...	approximately at am in circumstances that the ...
5	During the unloading operation of the ustulado...	during the unloading operation of the ustulado...
6	The collaborator reports that he was on street...	the collaborator reports that he was on street...

	Cleaned_Description	Cleaned_Description_without_stopwords
0	while removing the drill rod of the jumbo for ...	removing drill rod jumbo maintenance superviso...
1	during the activation of a sodium sulphide pum...	activation sodium sulphide pump piping uncoupl...
2	in the substation milpo located at level when ...	substation milpo located level collaborator ex...
3	being am approximately in the nv cx ob the per...	approximately nv cx ob personnel begins task u...
4	approximately at am in circumstances that the ...	approximately circumstances mechanics anthony ...
5	during the unloading operation of the ustulado...	unloading operation ustulado bag need unclog d...
6	the collaborator reports that he was on street...	collaborator reports street holding left hand ...

	Cleaned_Description_without_stopwords	final_cleaned_description
0	removing drill rod jumbo maintenance superviso...	remov drill rod jumbo mainten supervisor proce...
1	activation sodium sulphide pump piping uncoupl...	activ sodium sulphid pump pipe uncoupl sulfid ...
2	substation milpo located level collaborator ex...	substat milpo locat level collabor excav work ...
3	approximately nv cx ob personnel begins task u...	approxim nv cx ob personnel begin task unlock ...
4	approximately circumstances mechanics anthony ...	approxim circumst mechan anthoni group leader ...
5	unloading operation ustulado bag need unclog d...	unload oper ustulado bag need unclog discharg ...
6	collaborator reports street holding left hand ...	collabor report street hold left hand volumetr...

Dataset	Percentage Split
Train Set	60%
Validation Set	20%
Test Set	20%
Total	100%

	Country	Local	Industry Sector	Gender	Employee Type	Critical Risk	Description	Year	Month	Day	Accident Category	Cleaned_Description	Cleaned_Description_without_stopwords	final_cleaned_description	Accident_Level_encoded	Word2Vec_Embedding	GloVe_Embedding	SentenceTransformer_Embedding	TFIDF_Embedding
0	0	0	1	1	1	20	While removing the drill rod of the Jumbo 08 f...	2016	1	Friday	1	while removing the drill rod of the jumbo for ...	removing drill rod jumbo maintenance superviso...	remov drill rod jumbo mainten supervisor proce...	0	[0.0049478794, -0.004582413, 0.003834784, 0.01...	[-0.14254056, 0.1762849, 0.022502007, -0.19460...	[-0.058391057, -0.01851519, -0.0026409973, -0....	[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...
1	1	1	1	1	0	21	During the activation of a sodium sulphide pum...	2016	1	Saturday	1	during the activation of a sodium sulphide pum...	activation sodium sulphide pump piping uncoupl...	activ sodium sulphid pump pipe uncoupl sulfid ...	0	[0.001964574, -0.0027313854, 0.001979902, 0.00...	[-0.16578694, 0.20287126, -0.07683354, -0.0456...	[-0.05031255, 0.038256533, 0.02560695, -0.0467...	[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...
2	0	2	1	1	2	14	In the sub-station MILPO located at level +170...	2016	1	Wednesday	1	in the substation milpo located at level when ...	substation milpo located level collaborator ex...	substat milpo locat level collabor excav work ...	0	[0.0038760058, -0.0035065245, 0.0039448733, 0....	[-0.1420229, 0.14836252, 0.028008433, -0.17408...	[-0.080239795, 0.001987809, -0.019875638, -0.0...	[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...
3	0	3	1	1	1	16	Being 9:45 am. approximately in the Nv. 1880 C...	2016	1	Friday	1	being am approximately in the nv cx ob the per...	approximately nv cx ob personnel begins task u...	approxim nv cx ob personnel begin task unlock ...	0	[0.0039243456, -0.0038393158, 0.0032891822, 0....	[-0.1388698, -0.040865235, 0.0032027573, -0.12...	[-0.07879221, 0.060530663, -0.06068262, -0.070...	[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...
4	0	3	1	1	1	16	Approximately at 11:45 a.m. in circumstances t...	2016	1	Sunday	0	approximately at am in circumstances that the ...	approximately circumstances mechanics anthony ...	approxim circumst mechan anthoni group leader ...	3	[0.0033181403, -0.0036052873, 0.0030573579, 0....	[-0.1269299, 0.031568, -0.055016093, -0.204989...	[-0.1158196, 0.06217271, -0.038862057, -0.0093...	[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...

	Unnamed: 0	Data	Countries	Local	Industry Sector	Accident Level	Potential Accident Level	Genre	Employee or Third Party	Critical Risk	Description
0	0	2016-01-01	Country_01	Local_01	Mining	I	IV	Male	Third Party	Pressed	While removing the drill rod of the Jumbo 08 f...
1	1	2016-01-02	Country_02	Local_02	Mining	I	IV	Male	Employee	Pressurized Systems	During the activation of a sodium sulphide pum...
2	2	2016-01-06	Country_01	Local_03	Mining	I	III	Male	Third Party (Remote)	Manual Tools	In the sub-station MILPO located at level +170...
3	3	2016-01-08	Country_01	Local_04	Mining	I	I	Male	Third Party	Others	Being 9:45 am. approximately in the Nv. 1880 C...
4	4	2016-01-10	Country_01	Local_04	Mining	IV	IV	Male	Third Party	Others	Approximately at 11:45 a.m. in circumstances t...

	Unnamed: 0	Data	Countries	Local	Industry Sector	Accident Level	Potential Accident Level	Genre	Employee or Third Party	Critical Risk	Description
count	425.000000	425	425	425	425	425	425	425	425	425	425
unique	NaN	NaN	3	12	3	5	6	2	3	33	411
top	NaN	NaN	Country_01	Local_03	Mining	I	IV	Male	Third Party	Others	On 02/03/17 during the soil sampling in the re...
freq	NaN	NaN	251	90	241	316	143	403	189	232	3
mean	224.084706	2016-09-20 16:46:18.352941312	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
min	0.000000	2016-01-01 00:00:00	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
25%	118.000000	2016-05-01 00:00:00	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
50%	226.000000	2016-09-13 00:00:00	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
75%	332.000000	2017-02-08 00:00:00	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
max	438.000000	2017-07-09 00:00:00	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
std	125.526786	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN

	Data	Countries	Local	Industry Sector	Accident Level	Potential Accident Level	Genre	Employee or Third Party	Critical Risk	Description
77	2016-04-01	Country_01	Local_01	Mining	I	V	Male	Third Party (Remote)	Others	In circumstances that two workers of the Abrat...
262	2016-12-01	Country_01	Local_03	Mining	I	IV	Male	Employee	Others	During the activity of chuteo of ore in hopper...
303	2017-01-21	Country_02	Local_02	Mining	I	I	Male	Third Party (Remote)	Others	Employees engaged in the removal of material f...
345	2017-03-02	Country_03	Local_10	Others	I	I	Male	Third Party	Venomous Animals	On 02/03/17 during the soil sampling in the re...
346	2017-03-02	Country_03	Local_10	Others	I	I	Male	Third Party	Venomous Animals	On 02/03/17 during the soil sampling in the re...
355	2017-03-15	Country_03	Local_10	Others	I	I	Male	Third Party	Venomous Animals	Team of the VMS Project performed soil collect...
397	2017-05-23	Country_01	Local_04	Mining	I	IV	Male	Third Party	Projection of fragments	In moments when the 02 collaborators carried o...

	Embedding Type	Training Samples	Validation Samples	Testing Samples	Training Shape	Validation Shape	Testing Shape
0	Word2Vec	250	84	84	(250, 200)	(84, 200)	(84, 200)
1	GloVe	250	84	84	(250, 100)	(84, 100)	(84, 100)
2	Sentence Transformer	250	84	84	(250, 384)	(84, 384)	(84, 384)
3	TF-IDF	250	84	84	(250, 2272)	(84, 2272)	(84, 2272)
4	BoW	250	84	84	(250, 1)	(84, 1)	(84, 1)

		Before Sampling	After Sampling
Embedding Type	Dataset
BoW	Test	84	186
	Train	250	555
	Val	84	186
GloVe	Test	84	186
	Train	250	555
	Val	84	186
Sentence Transformer	Test	84	186
	Train	250	555
	Val	84	186
TF-IDF	Test	84	186
	Train	250	555
	Val	84	186
Word2Vec	Test	84	186
	Train	250	555
	Val	84	186

	Class Distribution	Test Before Sampling	Train Before Sampling	Val Before Sampling	Test After Sampling	Train After Sampling	Val After Sampling
0	0	8	22	8	62	185	62
1	1	62	185	62	62	185	62
2	2	14	43	14	62	185	62
3	Total	84	250	84	186	555	186

	Model	Type	Train Acc	Train Prec	Train Recall	Train F1	Val Acc	Val Prec	Val Recall	Val F1	Test Acc	Test Prec	Test Recall	Test F1
0	RandomForest	Base	0.9920	0.9921	0.9920	0.9918	0.7381	0.8067	0.7381	0.6269	0.7143	0.5401	0.7143	0.6151
1	DecisionTree	Base	0.9920	0.9927	0.9920	0.9921	0.4881	0.5075	0.4881	0.4973	0.5476	0.5696	0.5476	0.5571
2	NaiveBayes	Base	0.1880	0.7934	0.1880	0.0894	0.1548	0.7818	0.1548	0.0665	0.2143	0.7927	0.2143	0.1301
3	AdaBoost	Base	0.7520	0.7640	0.7520	0.7442	0.6310	0.5681	0.6310	0.5970	0.6429	0.5684	0.6429	0.6028
4	GradientBoost	Base	0.9920	0.9921	0.9920	0.9918	0.7024	0.6273	0.7024	0.6419	0.7143	0.6988	0.7143	0.6477
5	LogisticRegression	Base	0.7400	0.8076	0.7400	0.6294	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
6	KNN	Base	0.7320	0.6909	0.7320	0.6835	0.7381	0.6994	0.7381	0.7051	0.6548	0.6227	0.6548	0.6353
7	SVM	Base	0.7400	0.8076	0.7400	0.6294	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
8	XGBoost	Base	0.9920	0.9920	0.9920	0.9920	0.7143	0.6353	0.7143	0.6151	0.6786	0.6091	0.6786	0.6384

	Model	Type	Train Acc	Train Prec	Train Recall	Train F1	Val Acc	Val Prec	Val Recall	Val F1	Test Acc	Test Prec	Test Recall	Test F1
0	RandomForest	Base	0.9964	0.9964	0.9964	0.9964	0.4570	0.5896	0.4570	0.3913	0.4624	0.3487	0.4624	0.3733
1	DecisionTree	Base	0.9964	0.9964	0.9964	0.9964	0.4516	0.4434	0.4516	0.4380	0.5269	0.5639	0.5269	0.5160
2	NaiveBayes	Base	0.4613	0.6373	0.4613	0.3830	0.4355	0.6218	0.4355	0.3553	0.4946	0.6600	0.4946	0.4101
3	AdaBoost	Base	0.7856	0.8190	0.7856	0.7902	0.4462	0.5184	0.4462	0.4274	0.3710	0.2740	0.3710	0.3038
4	GradientBoost	Base	0.9964	0.9964	0.9964	0.9964	0.5161	0.5837	0.5161	0.5089	0.4355	0.3317	0.4355	0.3535
5	LogisticRegression	Base	0.4486	0.3957	0.4486	0.3925	0.4409	0.4604	0.4409	0.3685	0.4892	0.4260	0.4892	0.4196
6	KNN	Base	0.7730	0.8136	0.7730	0.7376	0.4731	0.4838	0.4731	0.4504	0.3548	0.3644	0.3548	0.3519
7	SVM	Base	0.4811	0.6540	0.4811	0.3848	0.4409	0.6280	0.4409	0.3520	0.5054	0.6703	0.5054	0.4043
8	XGBoost	Base	0.9964	0.9964	0.9964	0.9964	0.5376	0.6421	0.5376	0.5288	0.4247	0.4545	0.4247	0.3557

	Model	Type	Train Acc	Train Prec	Train Recall	Train F1	Val Acc	Val Prec	Val Recall	Val F1	Test Acc	Test Prec	Test Recall	Test F1
0	AdaBoost	Base	0.8200	0.7954	0.8200	0.7991	0.6786	0.5978	0.6786	0.6317	0.6548	0.5624	0.6548	0.6020
1	AdaBoost	Tuned	0.7400	0.8076	0.7400	0.6294	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
2	DecisionTree	Base	0.9920	0.9927	0.9920	0.9921	0.5238	0.5486	0.5238	0.5359	0.5952	0.5272	0.5952	0.5592
3	DecisionTree	Tuned	0.7720	0.7789	0.7720	0.7042	0.6786	0.6278	0.6786	0.5968	0.6905	0.6834	0.6905	0.6324
4	GradientBoost	Base	0.9920	0.9920	0.9920	0.9920	0.6667	0.6252	0.6667	0.5905	0.6905	0.5351	0.6905	0.6030
5	GradientBoost	Tuned	0.7400	0.8076	0.7400	0.6294	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
6	KNN	Base	0.7400	0.7353	0.7400	0.6962	0.6548	0.5843	0.6548	0.6173	0.5952	0.5427	0.5952	0.5678
7	KNN	Tuned	0.7480	0.7216	0.7480	0.6804	0.7143	0.5750	0.7143	0.6336	0.7262	0.7134	0.7262	0.6416
8	LogisticRegression	Base	0.7760	0.7708	0.7760	0.7113	0.7262	0.6975	0.7262	0.6390	0.7024	0.6705	0.7024	0.6248
9	LogisticRegression	Tuned	0.7400	0.8076	0.7400	0.6294	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
10	NaiveBayes	Base	0.7200	0.7751	0.7200	0.7371	0.6190	0.6761	0.6190	0.6417	0.6429	0.6363	0.6429	0.6395
11	RandomForest	Base	0.9920	0.9920	0.9920	0.9920	0.7262	0.6377	0.7262	0.6210	0.7381	0.8067	0.7381	0.6269
12	RandomForest	Tuned	0.8240	0.8311	0.8240	0.7825	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
13	SVM	Base	0.7400	0.8076	0.7400	0.6294	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
14	SVM	Tuned	0.7400	0.8076	0.7400	0.6294	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
15	XGBoost	Base	0.9920	0.9921	0.9920	0.9918	0.7143	0.6353	0.7143	0.6151	0.7143	0.6353	0.7143	0.6151

	Model	Type	Train Acc	Train Prec	Train Recall	Train F1	Val Acc	Val Prec	Val Recall	Val F1	Test Acc	Test Prec	Test Recall	Test F1
0	AdaBoost	Base	0.8440	0.8398	0.8440	0.8340	0.7262	0.6881	0.7262	0.6574	0.6905	0.5797	0.6905	0.6229
1	AdaBoost	Tuned	0.7520	0.8143	0.7520	0.6562	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
2	DecisionTree	Base	0.9920	0.9927	0.9920	0.9921	0.6667	0.6277	0.6667	0.6415	0.6071	0.5920	0.6071	0.5994
3	DecisionTree	Tuned	0.7840	0.8213	0.7840	0.7190	0.7143	0.6353	0.7143	0.6151	0.7381	0.8067	0.7381	0.6269
4	GradientBoost	Base	0.9920	0.9921	0.9920	0.9918	0.7143	0.6420	0.7143	0.6194	0.6786	0.5325	0.6786	0.5968
5	GradientBoost	Tuned	0.9920	0.9920	0.9920	0.9920	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
6	KNN	Base	0.7600	0.7167	0.7600	0.7181	0.6905	0.5488	0.6905	0.6116	0.6786	0.5701	0.6786	0.6123
7	KNN	Tuned	0.7320	0.7029	0.7320	0.6441	0.7262	0.7091	0.7262	0.6210	0.7381	0.6392	0.7381	0.6505
8	LogisticRegression	Base	0.7520	0.8143	0.7520	0.6562	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
9	LogisticRegression	Tuned	0.7400	0.8076	0.7400	0.6294	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
10	NaiveBayes	Base	0.8360	0.8770	0.8360	0.8451	0.6310	0.6210	0.6310	0.6253	0.6429	0.5920	0.6429	0.6162
11	RandomForest	Base	0.9920	0.9921	0.9920	0.9918	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
12	RandomForest	Tuned	0.8560	0.8795	0.8560	0.8251	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
13	SVM	Base	0.7560	0.8165	0.7560	0.6645	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
14	SVM	Tuned	0.7400	0.8076	0.7400	0.6294	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
15	XGBoost	Base	0.9920	0.9921	0.9920	0.9918	0.7381	0.6997	0.7381	0.6527	0.7381	0.7276	0.7381	0.6462

	Model	Type	Train Acc	Train Prec	Train Recall	Train F1	Val Acc	Val Prec	Val Recall	Val F1	Test Acc	Test Prec	Test Recall	Test F1
0	AdaBoost	Base	0.8775	0.8858	0.8775	0.8794	0.3925	0.5160	0.3925	0.3733	0.3602	0.4102	0.3602	0.3129
1	AdaBoost	Tuned	0.9766	0.9766	0.9766	0.9766	0.4301	0.6172	0.4301	0.3995	0.4032	0.5204	0.4032	0.3480
2	DecisionTree	Base	0.9964	0.9964	0.9964	0.9964	0.3387	0.3457	0.3387	0.2935	0.3387	0.3034	0.3387	0.2745
3	DecisionTree	Tuned	0.9730	0.9734	0.9730	0.9730	0.3656	0.3882	0.3656	0.3389	0.3441	0.2971	0.3441	0.2779
4	GradientBoost	Base	0.9964	0.9964	0.9964	0.9964	0.3602	0.5658	0.3602	0.2376	0.3656	0.4471	0.3656	0.2752
5	GradientBoost	Tuned	0.9964	0.9964	0.9964	0.9964	0.3548	0.7802	0.3548	0.2098	0.3441	0.3655	0.3441	0.2012
6	KNN	Base	0.7279	0.8069	0.7279	0.6608	0.3441	0.5371	0.3441	0.3010	0.4624	0.6246	0.4624	0.3887
7	KNN	Tuned	0.9964	0.9964	0.9964	0.9964	0.3548	0.5382	0.3548	0.3250	0.4516	0.4255	0.4516	0.4083
8	LogisticRegression	Base	0.9063	0.9066	0.9063	0.9049	0.5161	0.5418	0.5161	0.4777	0.3763	0.4222	0.3763	0.3457
9	LogisticRegression	Tuned	0.9820	0.9822	0.9820	0.9819	0.4677	0.5653	0.4677	0.4164	0.3602	0.4244	0.3602	0.3078
10	NaiveBayes	Base	0.8667	0.8766	0.8667	0.8687	0.5430	0.5994	0.5430	0.4940	0.4086	0.4594	0.4086	0.3426
11	RandomForest	Base	0.9964	0.9964	0.9964	0.9964	0.3710	0.7821	0.3710	0.2391	0.3495	0.4488	0.3495	0.2023
12	RandomForest	Tuned	0.9964	0.9964	0.9964	0.9964	0.4140	0.7497	0.4140	0.3055	0.3656	0.4501	0.3656	0.2318
13	SVM	Base	0.9838	0.9839	0.9838	0.9838	0.5269	0.6994	0.5269	0.4557	0.3495	0.3989	0.3495	0.2191
14	SVM	Tuned	0.9892	0.9894	0.9892	0.9892	0.3387	0.7784	0.3387	0.1779	0.3495	0.3815	0.3495	0.2100
15	XGBoost	Base	0.9964	0.9964	0.9964	0.9964	0.3548	0.3326	0.3548	0.2543	0.4355	0.6142	0.4355	0.3660

	Model	Type	Train Acc	Train Prec	Train Recall	Train F1	Val Acc	Val Prec	Val Recall	Val F1	Test Acc	Test Prec	Test Recall	Test F1
0	AdaBoost	Base	0.7207	0.7773	0.7207	0.7290	0.5000	0.5663	0.5000	0.4763	0.3710	0.5246	0.3710	0.3151
1	AdaBoost	Tuned	0.9225	0.9288	0.9225	0.9228	0.4409	0.3928	0.4409	0.3580	0.4516	0.4079	0.4516	0.3707
2	DecisionTree	Base	0.9964	0.9964	0.9964	0.9964	0.3495	0.3170	0.3495	0.3136	0.5108	0.6040	0.5108	0.4993
3	DecisionTree	Tuned	0.9838	0.9842	0.9838	0.9838	0.4570	0.5930	0.4570	0.4359	0.4140	0.5689	0.4140	0.3480
4	GradientBoost	Base	0.9964	0.9964	0.9964	0.9964	0.3710	0.3435	0.3710	0.3122	0.4301	0.4249	0.4301	0.3536
5	GradientBoost	Tuned	0.9964	0.9964	0.9964	0.9964	0.4677	0.7689	0.4677	0.3883	0.3710	0.7445	0.3710	0.2453
6	KNN	Base	0.6847	0.7858	0.6847	0.5854	0.3118	0.5359	0.3118	0.2542	0.4624	0.6390	0.4624	0.3881
7	KNN	Tuned	0.9964	0.9964	0.9964	0.9964	0.3925	0.4080	0.3925	0.2899	0.4032	0.6450	0.4032	0.3262
8	LogisticRegression	Base	0.9874	0.9875	0.9874	0.9874	0.3387	0.2983	0.3387	0.2282	0.3226	0.2197	0.3226	0.1807
9	LogisticRegression	Tuned	0.9964	0.9964	0.9964	0.9964	0.3495	0.3175	0.3495	0.2449	0.3226	0.2197	0.3226	0.1807
10	NaiveBayes	Base	0.9964	0.9964	0.9964	0.9964	0.3387	0.5092	0.3387	0.2121	0.3333	0.4413	0.3333	0.2094
11	RandomForest	Base	0.9964	0.9964	0.9964	0.9964	0.3817	0.7834	0.3817	0.2574	0.3333	0.7778	0.3333	0.1667
12	RandomForest	Tuned	0.9910	0.9911	0.9910	0.9910	0.3387	0.7784	0.3387	0.1779	0.3333	0.7778	0.3333	0.1667
13	SVM	Base	0.9946	0.9946	0.9946	0.9946	0.3387	0.7784	0.3387	0.1779	0.3387	0.7784	0.3387	0.1779
14	SVM	Tuned	0.9964	0.9964	0.9964	0.9964	0.3333	0.7778	0.3333	0.1667	0.3333	0.7778	0.3333	0.1667
15	XGBoost	Base	0.9964	0.9964	0.9964	0.9964	0.4570	0.5891	0.4570	0.4126	0.5108	0.5933	0.5108	0.4754

	Model	Type	Train Acc	Train Prec	Train Recall	Train F1	Val Acc	Val Prec	Val Recall	Val F1	Test Acc	Test Prec	Test Recall	Test F1
0	RandomForest	Base	0.7800	0.7802	0.7800	0.7367	0.6429	0.5906	0.6429	0.6141	0.7143	0.6617	0.7143	0.6728
1	DecisionTree	Base	0.7800	0.7865	0.7800	0.7262	0.6548	0.5694	0.6548	0.6085	0.7381	0.6950	0.7381	0.6904
2	NaiveBayes	Base	0.7400	0.7665	0.7400	0.6419	0.7262	0.7451	0.7262	0.6367	0.7024	0.7110	0.7024	0.6133
3	AdaBoost	Base	0.7400	0.8076	0.7400	0.6294	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
4	GradientBoost	Base	0.7800	0.7926	0.7800	0.7262	0.6429	0.5668	0.6429	0.6019	0.7381	0.6950	0.7381	0.6904
5	LogisticRegression	Base	0.7400	0.8076	0.7400	0.6294	0.7381	0.8067	0.7381	0.6269	0.7381	0.8067	0.7381	0.6269
6	KNN	Base	0.7480	0.7462	0.7480	0.6605	0.7024	0.5443	0.7024	0.6133	0.7500	0.7225	0.7500	0.6714
7	SVM	Base	0.7440	0.8098	0.7440	0.6385	0.7381	0.8067	0.7381	0.6269	0.7262	0.7091	0.7262	0.6210
8	XGBoost	Base	0.7760	0.7627	0.7760	0.7306	0.6429	0.5386	0.6429	0.5861	0.7262	0.6848	0.7262	0.6827

	Model	Type	Train Acc	Train Prec	Train Recall	Train F1	Val Acc	Val Prec	Val Recall	Val F1	Test Acc	Test Prec	Test Recall	Test F1
0	RandomForest	Base	0.8288	0.8302	0.8288	0.8267	0.2849	0.2806	0.2849	0.2686	0.3656	0.3534	0.3656	0.3403
1	DecisionTree	Base	0.8288	0.8292	0.8288	0.8267	0.2849	0.2715	0.2849	0.2623	0.3763	0.3585	0.3763	0.3493
2	NaiveBayes	Base	0.3748	0.3844	0.3748	0.3154	0.3333	0.2712	0.3333	0.2915	0.2957	0.2891	0.2957	0.2679
3	AdaBoost	Base	0.6144	0.6034	0.6144	0.5873	0.2258	0.2399	0.2258	0.2310	0.2742	0.2787	0.2742	0.2615
4	GradientBoost	Base	0.8036	0.8073	0.8036	0.8022	0.2796	0.2725	0.2796	0.2641	0.3817	0.3761	0.3817	0.3584
5	LogisticRegression	Base	0.3892	0.5959	0.3892	0.3117	0.4624	0.6368	0.4624	0.3641	0.3763	0.5837	0.3763	0.3007
6	KNN	Base	0.6649	0.6998	0.6649	0.6445	0.3602	0.3029	0.3602	0.3128	0.4516	0.4787	0.4516	0.4220
7	SVM	Base	0.5099	0.5461	0.5099	0.4704	0.2634	0.4230	0.2634	0.2568	0.3065	0.3335	0.3065	0.2843
8	XGBoost	Base	0.8180	0.8192	0.8180	0.8156	0.3065	0.3093	0.3065	0.2938	0.3441	0.3319	0.3441	0.3225

	Description	Accident Category	processed_text
0	While removing the drill rod of the Jumbo 08 f...	1	removing drill rod jumbo 08 maintenance superv...
1	During the activation of a sodium sulphide pum...	1	activation sodium sulphide pump piping uncoupl...
2	In the sub-station MILPO located at level +170...	1	sub-station milpo located level +170 collabora...
3	Being 9:45 am. approximately in the Nv. 1880 C...	1	9:45 approximately nv 1880 cx-695 ob7 personne...
4	Approximately at 11:45 a.m. in circumstances t...	0	approximately 11:45 a.m. circumstance mechanic...

	Train	Validation	Test
Accuracy	0.727273	0.781250	0.781250
Recall	0.727273	0.781250	0.781250
Precision	0.528926	0.610352	0.610352
F1_score	0.612440	0.685307	0.685307

	Train	Validation	Test
Accuracy	0.740741	0.765625	0.781250
Recall	0.740741	0.765625	0.781250
Precision	0.742525	0.617440	0.610352
F1_score	0.644627	0.683594	0.685307

	Train	Validation	Test
Accuracy	0.747475	0.796875	0.765625
Recall	0.747475	0.796875	0.765625
Precision	0.691765	0.721311	0.607639
F1_score	0.656523	0.736627	0.677544

	Train	Validation	Test
Accuracy	0.808081	0.781250	0.78125
Recall	0.808081	0.781250	0.78125
Precision	0.820029	0.630040	0.73976
F1_score	0.762263	0.697545	0.73615

	Train	Validation	Test
Accuracy	0.905724	0.781250	0.734375
Recall	0.905724	0.781250	0.734375
Precision	0.911585	0.630040	0.679276
F1_score	0.898401	0.697545	0.698793

Natural Language Processing¶

Importing the libraries¶

MILESTONE 1¶

Problem Definition¶

Loading the Dataset¶

Taking a look at the Columns and their Datatypes

Five Points summary of our dataset

Cleaning the Dataset¶

Dropping Irrelevant Columns

Checking Missing or Duplicate Values

Standardizing the Columns¶

Exploratory Data Analysis¶

Univariate Analysis

Insights for the above plots¶

How Bad Are the Accidents?¶

When Do Accidents Happen?¶

Where Are Most Incidents Reported?¶

Bivariate Analysis

Distribution of other features with respect to different countries.¶

Distribution of other features with respect to Gender.¶

Gender Counts by Critical Risk¶

Distribution of Accidents over different years¶

Multivariate Analysis

Pair Plot for all Numerical Variables¶

Correlation Heatmap for all Numerical Variables¶

Feature Engineering¶

Handling Class Imbalance¶

NLP Text Processing¶

Removing special characters from the text¶

Removing stopwords¶

Stemming¶

Dropping Irrelevant Columns

Label Encoding¶

Split the Target Variable and Predictors¶

Split the Data into train, validation, and test sets¶

Word Embeddings¶

Using Word2Vec¶

Using GloVe¶

Using Sentence Transformer¶

Using TF-IDF (Term Frequency - Inverse Document Frequency)¶

Using Bag of Words (BoW)¶

Training Basic Models - All (Word2Vec)¶

Tuning the Models - All (Word2Vec)¶

Basic Models (Resampled) - All (Word2Vec)¶

Tuning the Models (Resampled) - All (Word2Vec)¶

Basic Models - All (GloVe)¶

Tuning the Models - All (GloVe)¶

Basic Models (Resampled) - All (GloVe)¶

Tuning the Models (Resampled) - All (GloVe)¶

Basic Models - All (Sentence Transformer)¶

Tuning the Models - All (Sentence Transformer)¶

Basic Models (Resampled) - All (Sentence Transformer)¶

Tuning the Models (Resampled) - All (Sentence Transformer)¶

Basic Models - All (TF-IDF)¶

Tuning the Models - All (TF-IDF)¶

Basic Models (Resampled) - All (TF-IDF)¶

Tuning the Models (Resampled) - All (TF-IDF)¶

Basic Models - All (BoW)¶

Tuning the Models - All (BoW)¶

Basic Models (Resampled) - All (BoW)¶

Tuning the Models (Resampled) - All (BoW)¶

Choose the Best Model from the ones built with Proper Reasoning¶

Key Observations:¶

Check the Performance of the Final Model on Train / Test / Validation Sets¶

Analysis & Recommendation from Above¶

-------------------------------------- END OF MILESTONE 1 --------------------------------------¶

MILESTONE 2¶

Transition from Milestone 1 to Milestone 2¶

Data Preprocessing (NLP Preprocessing Techniques)¶

Word Embeddings¶

Word2Vec Embedding¶

GloVe Embedding¶

Sentence Transformer Embedding¶

Design, Train, and Test Neural Networks Classifiers¶

Training Basic Neural Network (Word2Vec Embedding)¶

Tuning Neural Network (Word2Vec Embedding)¶

Training Basic Neural Network (GloVe Embedding)¶

Tuning Basic Neural Network (GloVe Embedding)¶

Training Basic Neural Network (Sentence Transformer Embedding)¶

Tuning Basic Neural Network (Sentence Transformer Embedding)¶

	Train	Validation	Test
Accuracy	0.831790	0.359375	0.296875
Recall	0.831790	0.359375	0.296875
Precision	0.885596	0.649609	0.656072
F1_score	0.822432	0.397321	0.342487

	Train	Validation	Test
Accuracy	0.364198	0.109375	0.093750
Recall	0.364198	0.109375	0.093750
Precision	0.781316	0.527237	0.785348
F1_score	0.228714	0.070799	0.067788

	Train	Validation	Test
Accuracy	0.905864	0.562500	0.500000
Recall	0.905864	0.562500	0.500000
Precision	0.917710	0.719188	0.694304
F1_score	0.904167	0.610417	0.556603

	Train	Validation	Test
Accuracy	0.938272	0.593750	0.500000
Recall	0.938272	0.593750	0.500000
Precision	0.938630	0.689453	0.632792
F1_score	0.937494	0.629936	0.553618

	Train	Validation	Test
Accuracy	0.993827	0.687500	0.656250
Recall	0.993827	0.687500	0.656250
Precision	0.993891	0.708163	0.673895
F1_score	0.993827	0.695466	0.664872

	Train	Validation	Test
Accuracy	0.995370	0.703125	0.640625
Recall	0.995370	0.703125	0.640625
Precision	0.995434	0.732304	0.671845
F1_score	0.995377	0.716561	0.655529

Epoch	Training Loss	Validation Loss	Accuracy	Precision	Recall	F1
1	1.962200	2.072852	0.156250	0.024414	0.156250	0.042230
2	1.099300	0.970368	0.781250	0.610352	0.781250	0.685307
3	0.845500	0.707521	0.781250	0.610352	0.781250	0.685307

	Train	Validation	Test
Accuracy	0.7273	0.7812	0.7812
Precision	0.5289	0.6104	0.6104
Recall	0.7273	0.7812	0.7812
F1_score	0.6124	0.6853	0.6853

Epoch	Training Loss	Validation Loss	Accuracy	Precision	Recall	F1
1	1.683700	1.500896	0.062500	0.003906	0.062500	0.007353
2	1.362600	1.658767	0.156250	0.024414	0.156250	0.042230
3	1.252700	0.933511	0.781250	0.610352	0.781250	0.685307

Epoch	Training Loss	Validation Loss	Accuracy	Precision	Recall	F1
1	0.986200	0.737886	0.781250	0.610352	0.781250	0.685307
2	0.837700	0.700930	0.781250	0.610352	0.781250	0.685307
3	0.717900	0.683717	0.781250	0.610352	0.781250	0.685307